From a8a800af51a55a27bfad29c00a1f528e9a0ba4f2 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Tue, 24 Oct 2023 12:12:11 +0100 Subject: [PATCH 01/69] Run real Azure tests on CI (#5627) ## Problem We do not run real Azure-related tests on CI ## Summary of changes - Set required env variables to run real Azure blob storage tests on CI --- .github/workflows/build_and_test.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 3f1c728c70..ffed67eb67 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -338,6 +338,16 @@ jobs: # Avoid `$CARGO_FEATURES` since there's no `testing` feature in the e2e tests now ${cov_prefix} cargo test $CARGO_FLAGS --package remote_storage --test test_real_s3 + # Run separate tests for real Azure Blob Storage + # XXX: replace region with `eu-central-1`-like region + export AZURE_STORAGE_ACCOUNT="${{ secrets.AZURE_STORAGE_ACCOUNT_DEV }}" + export AZURE_STORAGE_ACCESS_KEY="${{ secrets.AZURE_STORAGE_ACCESS_KEY_DEV }}" + export ENABLE_REAL_AZURE_REMOTE_STORAGE=y + export REMOTE_STORAGE_AZURE_CONTAINER=neon-github-sandbox + export REMOTE_STORAGE_AZURE_REGION=eastus2 + # Avoid `$CARGO_FEATURES` since there's no `testing` feature in the e2e tests now + ${cov_prefix} cargo test $CARGO_FLAGS --package remote_storage --test test_real_azure + - name: Install rust binaries run: | # Install target binaries From 767ef293902bcba71d888afad36931d8fc952222 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Tue, 24 Oct 2023 13:13:23 +0100 Subject: [PATCH 02/69] proxy: filter out more quota exceeded errors (#5640) ## Problem Looking at logs, I saw more retries being performed for other quota exceeded errors ## Summary of changes Filter out all quota exceeded family of errors --- proxy/src/console/provider.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/proxy/src/console/provider.rs b/proxy/src/console/provider.rs index 8742534f5d..86fd810ac3 100644 --- a/proxy/src/console/provider.rs +++ b/proxy/src/console/provider.rs @@ -90,7 +90,11 @@ pub mod errors { status: http::StatusCode::LOCKED, ref text, } => { - !text.contains("written data quota exceeded") + // written data quota exceeded + // data transfer quota exceeded + // compute time quota exceeded + // logical size quota exceeded + !text.contains("quota exceeded") && !text.contains("the limit for current plan reached") } // retry server errors From a0b862a8bd63bfbad4cf509774e47436ec79ebf0 Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 24 Oct 2023 13:57:01 +0100 Subject: [PATCH 03/69] pageserver: schedule frozen layer uploads inside the layers lock (#5639) ## Problem Compaction's source of truth for what layers exist is the LayerManager. `flush_frozen_layer` updates LayerManager before it has scheduled upload of the frozen layer. Compaction can then "see" the new layer, decide to delete it, schedule uploads of replacement layers, all before `flush_frozen_layer` wakes up again and schedules the upload. When the upload is scheduled, the local layer file may be gone, in which case we end up with no such layer in remote storage, but an entry still added to IndexPart pointing to the missing layer. ## Summary of changes Schedule layer uploads inside the `self.layers` lock, so that whenever a frozen layer is present in LayerManager, it is also present in RemoteTimelineClient's metadata. Closes: #5635 --- .../tenant/remote_timeline_client/upload.rs | 2 + pageserver/src/tenant/timeline.rs | 52 ++++++++---- test_runner/regress/test_remote_storage.py | 83 +++++++++++++------ 3 files changed, 94 insertions(+), 43 deletions(-) diff --git a/pageserver/src/tenant/remote_timeline_client/upload.rs b/pageserver/src/tenant/remote_timeline_client/upload.rs index 90e603deb0..03ba137566 100644 --- a/pageserver/src/tenant/remote_timeline_client/upload.rs +++ b/pageserver/src/tenant/remote_timeline_client/upload.rs @@ -60,6 +60,8 @@ pub(super) async fn upload_timeline_layer<'a>( bail!("failpoint before-upload-layer") }); + pausable_failpoint!("before-upload-layer-pausable"); + let storage_path = remote_path(conf, source_path, generation)?; let source_file_res = fs::File::open(&source_path).await; let source_file = match source_file_res { diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 2c76155e2a..8ce6aa2108 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -2793,10 +2793,13 @@ impl Timeline { ) }; + let disk_consistent_lsn = Lsn(lsn_range.end.0 - 1); + let old_disk_consistent_lsn = self.disk_consistent_lsn.load(); + // The new on-disk layers are now in the layer map. We can remove the // in-memory layer from the map now. The flushed layer is stored in // the mapping in `create_delta_layer`. - { + let metadata = { let mut guard = self.layers.write().await; if let Some(ref l) = delta_layer_to_add { @@ -2812,8 +2815,17 @@ impl Timeline { } guard.finish_flush_l0_layer(delta_layer_to_add, &frozen_layer); + if disk_consistent_lsn != old_disk_consistent_lsn { + assert!(disk_consistent_lsn > old_disk_consistent_lsn); + self.disk_consistent_lsn.store(disk_consistent_lsn); + + // Schedule remote uploads that will reflect our new disk_consistent_lsn + Some(self.schedule_uploads(disk_consistent_lsn, layer_paths_to_upload)?) + } else { + None + } // release lock on 'layers' - } + }; // FIXME: between create_delta_layer and the scheduling of the upload in `update_metadata_file`, // a compaction can delete the file and then it won't be available for uploads any more. @@ -2829,28 +2841,22 @@ impl Timeline { // // TODO: This perhaps should be done in 'flush_frozen_layers', after flushing // *all* the layers, to avoid fsyncing the file multiple times. - let disk_consistent_lsn = Lsn(lsn_range.end.0 - 1); - let old_disk_consistent_lsn = self.disk_consistent_lsn.load(); - // If we were able to advance 'disk_consistent_lsn', save it the metadata file. - // After crash, we will restart WAL streaming and processing from that point. - if disk_consistent_lsn != old_disk_consistent_lsn { - assert!(disk_consistent_lsn > old_disk_consistent_lsn); - self.update_metadata_file(disk_consistent_lsn, layer_paths_to_upload) + // If we updated our disk_consistent_lsn, persist the updated metadata to local disk. + if let Some(metadata) = metadata { + save_metadata(self.conf, &self.tenant_id, &self.timeline_id, &metadata) .await - .context("update_metadata_file")?; - // Also update the in-memory copy - self.disk_consistent_lsn.store(disk_consistent_lsn); + .context("save_metadata")?; } Ok(()) } /// Update metadata file - async fn update_metadata_file( + fn schedule_uploads( &self, disk_consistent_lsn: Lsn, layer_paths_to_upload: HashMap, - ) -> anyhow::Result<()> { + ) -> anyhow::Result { // We can only save a valid 'prev_record_lsn' value on disk if we // flushed *all* in-memory changes to disk. We only track // 'prev_record_lsn' in memory for the latest processed record, so we @@ -2887,10 +2893,6 @@ impl Timeline { x.unwrap() )); - save_metadata(self.conf, &self.tenant_id, &self.timeline_id, &metadata) - .await - .context("save_metadata")?; - if let Some(remote_client) = &self.remote_client { for (path, layer_metadata) in layer_paths_to_upload { remote_client.schedule_layer_file_upload(&path, &layer_metadata)?; @@ -2898,6 +2900,20 @@ impl Timeline { remote_client.schedule_index_upload_for_metadata_update(&metadata)?; } + Ok(metadata) + } + + async fn update_metadata_file( + &self, + disk_consistent_lsn: Lsn, + layer_paths_to_upload: HashMap, + ) -> anyhow::Result<()> { + let metadata = self.schedule_uploads(disk_consistent_lsn, layer_paths_to_upload)?; + + save_metadata(self.conf, &self.tenant_id, &self.timeline_id, &metadata) + .await + .context("save_metadata")?; + Ok(()) } diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py index ecd02c4c16..f4bf9207b0 100644 --- a/test_runner/regress/test_remote_storage.py +++ b/test_runner/regress/test_remote_storage.py @@ -757,12 +757,14 @@ def test_empty_branch_remote_storage_upload_on_restart(neon_env_builder: NeonEnv create_thread.join() -# Regression test for a race condition where L0 layers are compacted before the upload, -# resulting in the uploading complaining about the file not being found -# https://github.com/neondatabase/neon/issues/4526 -def test_compaction_delete_before_upload( +def test_compaction_waits_for_upload( neon_env_builder: NeonEnvBuilder, ): + """ + Compaction waits for outstanding uploads to complete, so that it avoids deleting layers + files that have not yet been uploaded. This test forces a race between upload and + compaction. + """ neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start( @@ -792,50 +794,81 @@ def test_compaction_delete_before_upload( wait_for_last_flush_lsn(env, endpoint, tenant_id, timeline_id) # Now make the flushing hang and update one small piece of data - client.configure_failpoints(("flush-frozen-pausable", "pause")) + client.configure_failpoints(("before-upload-layer-pausable", "pause")) endpoint.safe_psql("UPDATE foo SET x = 0 WHERE x = 1") wait_for_last_flush_lsn(env, endpoint, tenant_id, timeline_id) - q: queue.Queue[Optional[PageserverApiException]] = queue.Queue() - barrier = threading.Barrier(2) + checkpoint_result: queue.Queue[Optional[PageserverApiException]] = queue.Queue() + compact_result: queue.Queue[Optional[PageserverApiException]] = queue.Queue() + compact_barrier = threading.Barrier(2) def checkpoint_in_background(): - barrier.wait() try: + log.info("Checkpoint starting") client.timeline_checkpoint(tenant_id, timeline_id) - q.put(None) + log.info("Checkpoint complete") + checkpoint_result.put(None) except PageserverApiException as e: - q.put(e) + log.info("Checkpoint errored: {e}") + checkpoint_result.put(e) - create_thread = threading.Thread(target=checkpoint_in_background) - create_thread.start() + def compact_in_background(): + compact_barrier.wait() + try: + log.info("Compaction starting") + client.timeline_compact(tenant_id, timeline_id) + log.info("Compaction complete") + compact_result.put(None) + except PageserverApiException as e: + log.info("Compaction errored: {e}") + compact_result.put(e) + + checkpoint_thread = threading.Thread(target=checkpoint_in_background) + checkpoint_thread.start() + + compact_thread = threading.Thread(target=compact_in_background) + compact_thread.start() try: - barrier.wait() + # Start the checkpoint, see that it blocks + log.info("Waiting to see checkpoint hang...") + time.sleep(5) + assert checkpoint_result.empty() - time.sleep(4) - client.timeline_compact(tenant_id, timeline_id) + # Start the compaction, see that it finds work to do but blocks + compact_barrier.wait() + log.info("Waiting to see compaction hang...") + time.sleep(5) + assert compact_result.empty() - client.configure_failpoints(("flush-frozen-pausable", "off")) + # This is logged once compaction is started, but before we wait for operations to complete + assert env.pageserver.log_contains("compact_level0_phase1 stats available.") - conflict = q.get() + # Once we unblock uploads the compaction should complete successfully + log.info("Disabling failpoint") + client.configure_failpoints(("before-upload-layer-pausable", "off")) + log.info("Awaiting compaction result") + assert compact_result.get(timeout=10) is None + log.info("Awaiting checkpoint result") + assert checkpoint_result.get(timeout=10) is None - assert conflict is None + except Exception: + # Log the actual failure's backtrace here, before we proceed to join threads + log.exception("Failure, cleaning up...") + raise finally: - create_thread.join() + compact_barrier.abort() - # Add a delay for the uploads to run into either the file not found or the - time.sleep(4) + checkpoint_thread.join() + compact_thread.join() # Ensure that this actually terminates wait_upload_queue_empty(client, tenant_id, timeline_id) - # For now we are hitting this message. - # Maybe in the future the underlying race condition will be fixed, - # but until then, ensure that this message is hit instead. - assert env.pageserver.log_contains( + # We should not have hit the error handling path in uploads where the remote file is gone + assert not env.pageserver.log_contains( "File to upload doesn't exist. Likely the file has been deleted and an upload is not required any more." ) From 4bef977c56d9c798c1f9482a55f4742686e834b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Tue, 24 Oct 2023 19:16:23 +0200 Subject: [PATCH 04/69] Use tuples instead of manual comparison chain (#5637) Makes code a little bit simpler --- libs/pageserver_api/src/reltag.rs | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/libs/pageserver_api/src/reltag.rs b/libs/pageserver_api/src/reltag.rs index c98ad259bf..33402ca8ba 100644 --- a/libs/pageserver_api/src/reltag.rs +++ b/libs/pageserver_api/src/reltag.rs @@ -22,9 +22,9 @@ use postgres_ffi::Oid; /// [See more related comments here](https:///github.com/postgres/postgres/blob/99c5852e20a0987eca1c38ba0c09329d4076b6a0/src/include/storage/relfilenode.h#L57). /// // FIXME: should move 'forknum' as last field to keep this consistent with Postgres. -// Then we could replace the custo Ord and PartialOrd implementations below with -// deriving them. -#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy, Serialize, Deserialize)] +// Then we could replace the custom Ord and PartialOrd implementations below with +// deriving them. This will require changes in walredoproc.c. +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy, Serialize)] pub struct RelTag { pub forknum: u8, pub spcnode: Oid, @@ -40,21 +40,9 @@ impl PartialOrd for RelTag { impl Ord for RelTag { fn cmp(&self, other: &Self) -> Ordering { - let mut cmp = self.spcnode.cmp(&other.spcnode); - if cmp != Ordering::Equal { - return cmp; - } - cmp = self.dbnode.cmp(&other.dbnode); - if cmp != Ordering::Equal { - return cmp; - } - cmp = self.relnode.cmp(&other.relnode); - if cmp != Ordering::Equal { - return cmp; - } - cmp = self.forknum.cmp(&other.forknum); - - cmp + // Custom ordering where we put forknum to the end of the list + let other_tup = (other.spcnode, other.dbnode, other.relnode, other.forknum); + (self.spcnode, self.dbnode, self.relnode, self.forknum).cmp(&other_tup) } } From 44202eeb3bd4f0eaa4c0bb6e484e2e19b5482ed2 Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Tue, 24 Oct 2023 16:04:28 -0700 Subject: [PATCH 05/69] Bump vm-builder v0.18.1 -> v0.18.2 (#5646) Only applicable change was neondatabase/autoscaling#571, removing the postgres_exporter flags `--auto-discover-databases` and `--exclude-databases=...` --- .github/workflows/build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index ffed67eb67..9a3d352ad0 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -847,7 +847,7 @@ jobs: run: shell: sh -eu {0} env: - VM_BUILDER_VERSION: v0.18.1 + VM_BUILDER_VERSION: v0.18.2 steps: - name: Checkout From b2c96047d01f4be7935f421081e764e2e3cfbcb5 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Wed, 25 Oct 2023 08:30:47 +0100 Subject: [PATCH 06/69] move wake compute after the auth quirks logic (#5642) ## Problem https://github.com/neondatabase/neon/issues/5568#issuecomment-1777015606 ## Summary of changes Make the auth_quirks_creds return the authentication information, and push the wake_compute loop to after, inside `auth_quirks` --- proxy/src/auth/backend.rs | 60 ++++++++++++++++++++++++++++--- proxy/src/auth/backend/classic.rs | 42 +++++----------------- proxy/src/auth/backend/hacks.rs | 25 +++---------- 3 files changed, 69 insertions(+), 58 deletions(-) diff --git a/proxy/src/auth/backend.rs b/proxy/src/auth/backend.rs index aa19acf65a..850d3a1837 100644 --- a/proxy/src/auth/backend.rs +++ b/proxy/src/auth/backend.rs @@ -3,7 +3,9 @@ mod hacks; mod link; pub use link::LinkAuthError; +use tokio_postgres::config::AuthKeys; +use crate::proxy::{handle_try_wake, retry_after}; use crate::{ auth::{self, ClientCredentials}, config::AuthenticationConfig, @@ -16,8 +18,9 @@ use crate::{ }; use futures::TryFutureExt; use std::borrow::Cow; +use std::ops::ControlFlow; use tokio::io::{AsyncRead, AsyncWrite}; -use tracing::info; +use tracing::{error, info, warn}; /// A product of successful authentication. pub struct AuthSuccess { @@ -117,22 +120,27 @@ impl<'a, T, E> BackendType<'a, Result> { } } +pub enum ComputeCredentials { + Password(Vec), + AuthKeys(AuthKeys), +} + /// True to its name, this function encapsulates our current auth trade-offs. /// Here, we choose the appropriate auth flow based on circumstances. -async fn auth_quirks( +async fn auth_quirks_creds( api: &impl console::Api, extra: &ConsoleReqExtra<'_>, creds: &mut ClientCredentials<'_>, client: &mut stream::PqStream, allow_cleartext: bool, config: &'static AuthenticationConfig, -) -> auth::Result> { +) -> auth::Result> { // If there's no project so far, that entails that client doesn't // support SNI or other means of passing the endpoint (project) name. // We now expect to see a very specific payload in the place of password. if creds.project.is_none() { // Password will be checked by the compute node later. - return hacks::password_hack(api, extra, creds, client).await; + return hacks::password_hack(creds, client).await; } // Password hack should set the project name. @@ -143,13 +151,55 @@ async fn auth_quirks( // Currently, we use it for websocket connections (latency). if allow_cleartext { // Password will be checked by the compute node later. - return hacks::cleartext_hack(api, extra, creds, client).await; + return hacks::cleartext_hack(client).await; } // Finally, proceed with the main auth flow (SCRAM-based). classic::authenticate(api, extra, creds, client, config).await } +/// True to its name, this function encapsulates our current auth trade-offs. +/// Here, we choose the appropriate auth flow based on circumstances. +async fn auth_quirks( + api: &impl console::Api, + extra: &ConsoleReqExtra<'_>, + creds: &mut ClientCredentials<'_>, + client: &mut stream::PqStream, + allow_cleartext: bool, + config: &'static AuthenticationConfig, +) -> auth::Result> { + let auth_stuff = auth_quirks_creds(api, extra, creds, client, allow_cleartext, config).await?; + + let mut num_retries = 0; + let mut node = loop { + let wake_res = api.wake_compute(extra, creds).await; + match handle_try_wake(wake_res, num_retries) { + Err(e) => { + error!(error = ?e, num_retries, retriable = false, "couldn't wake compute node"); + return Err(e.into()); + } + Ok(ControlFlow::Continue(e)) => { + warn!(error = ?e, num_retries, retriable = true, "couldn't wake compute node"); + } + Ok(ControlFlow::Break(n)) => break n, + } + + let wait_duration = retry_after(num_retries); + num_retries += 1; + tokio::time::sleep(wait_duration).await; + }; + + match auth_stuff.value { + ComputeCredentials::Password(password) => node.config.password(password), + ComputeCredentials::AuthKeys(auth_keys) => node.config.auth_keys(auth_keys), + }; + + Ok(AuthSuccess { + reported_auth_ok: auth_stuff.reported_auth_ok, + value: node, + }) +} + impl BackendType<'_, ClientCredentials<'_>> { /// Get compute endpoint name from the credentials. pub fn get_endpoint(&self) -> Option { diff --git a/proxy/src/auth/backend/classic.rs b/proxy/src/auth/backend/classic.rs index af628efc5e..e7013a4cac 100644 --- a/proxy/src/auth/backend/classic.rs +++ b/proxy/src/auth/backend/classic.rs @@ -1,17 +1,14 @@ -use std::ops::ControlFlow; - -use super::AuthSuccess; +use super::{AuthSuccess, ComputeCredentials}; use crate::{ auth::{self, AuthFlow, ClientCredentials}, compute, config::AuthenticationConfig, - console::{self, AuthInfo, CachedNodeInfo, ConsoleReqExtra}, - proxy::{handle_try_wake, retry_after}, + console::{self, AuthInfo, ConsoleReqExtra}, sasl, scram, stream::PqStream, }; use tokio::io::{AsyncRead, AsyncWrite}; -use tracing::{error, info, warn}; +use tracing::{info, warn}; pub(super) async fn authenticate( api: &impl console::Api, @@ -19,7 +16,7 @@ pub(super) async fn authenticate( creds: &ClientCredentials<'_>, client: &mut PqStream, config: &'static AuthenticationConfig, -) -> auth::Result> { +) -> auth::Result> { info!("fetching user's authentication info"); let info = api.get_auth_info(extra, creds).await?.unwrap_or_else(|| { // If we don't have an authentication secret, we mock one to @@ -66,38 +63,17 @@ pub(super) async fn authenticate( } }; - Some(compute::ScramKeys { + compute::ScramKeys { client_key: client_key.as_bytes(), server_key: secret.server_key.as_bytes(), - }) + } } }; - let mut num_retries = 0; - let mut node = loop { - let wake_res = api.wake_compute(extra, creds).await; - match handle_try_wake(wake_res, num_retries) { - Err(e) => { - error!(error = ?e, num_retries, retriable = false, "couldn't wake compute node"); - return Err(e.into()); - } - Ok(ControlFlow::Continue(e)) => { - warn!(error = ?e, num_retries, retriable = true, "couldn't wake compute node"); - } - Ok(ControlFlow::Break(n)) => break n, - } - - let wait_duration = retry_after(num_retries); - num_retries += 1; - tokio::time::sleep(wait_duration).await; - }; - if let Some(keys) = scram_keys { - use tokio_postgres::config::AuthKeys; - node.config.auth_keys(AuthKeys::ScramSha256(keys)); - } - Ok(AuthSuccess { reported_auth_ok: false, - value: node, + value: ComputeCredentials::AuthKeys(tokio_postgres::config::AuthKeys::ScramSha256( + scram_keys, + )), }) } diff --git a/proxy/src/auth/backend/hacks.rs b/proxy/src/auth/backend/hacks.rs index dcc93ec04c..53958bf337 100644 --- a/proxy/src/auth/backend/hacks.rs +++ b/proxy/src/auth/backend/hacks.rs @@ -1,10 +1,6 @@ -use super::AuthSuccess; +use super::{AuthSuccess, ComputeCredentials}; use crate::{ auth::{self, AuthFlow, ClientCredentials}, - console::{ - self, - provider::{CachedNodeInfo, ConsoleReqExtra}, - }, stream, }; use tokio::io::{AsyncRead, AsyncWrite}; @@ -15,11 +11,8 @@ use tracing::{info, warn}; /// These properties are benefical for serverless JS workers, so we /// use this mechanism for websocket connections. pub async fn cleartext_hack( - api: &impl console::Api, - extra: &ConsoleReqExtra<'_>, - creds: &mut ClientCredentials<'_>, client: &mut stream::PqStream, -) -> auth::Result> { +) -> auth::Result> { warn!("cleartext auth flow override is enabled, proceeding"); let password = AuthFlow::new(client) .begin(auth::CleartextPassword) @@ -27,24 +20,19 @@ pub async fn cleartext_hack( .authenticate() .await?; - let mut node = api.wake_compute(extra, creds).await?; - node.config.password(password); - // Report tentative success; compute node will check the password anyway. Ok(AuthSuccess { reported_auth_ok: false, - value: node, + value: ComputeCredentials::Password(password), }) } /// Workaround for clients which don't provide an endpoint (project) name. /// Very similar to [`cleartext_hack`], but there's a specific password format. pub async fn password_hack( - api: &impl console::Api, - extra: &ConsoleReqExtra<'_>, creds: &mut ClientCredentials<'_>, client: &mut stream::PqStream, -) -> auth::Result> { +) -> auth::Result> { warn!("project not specified, resorting to the password hack auth flow"); let payload = AuthFlow::new(client) .begin(auth::PasswordHack) @@ -55,12 +43,9 @@ pub async fn password_hack( info!(project = &payload.endpoint, "received missing parameter"); creds.project = Some(payload.endpoint); - let mut node = api.wake_compute(extra, creds).await?; - node.config.password(payload.password); - // Report tentative success; compute node will check the password anyway. Ok(AuthSuccess { reported_auth_ok: false, - value: node, + value: ComputeCredentials::Password(payload.password), }) } From 127837abb0112ecbfed0b1901b4dd272f36e04a3 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 25 Oct 2023 10:55:45 +0100 Subject: [PATCH 07/69] tests: de-flake test_eviction_across_generations (#5650) ## Problem There was an edge case where initial logical size calculation can be downloading a layer that wasn't hit by the test's `SELECT`, and it's on-disk but still marked as remote in the pageserver's internal state, so evicting it fails. https://neon-github-public-dev.s3.amazonaws.com/reports/pr-5648/6630099807/index.html#categories/dee044ec96f666edb90a77c01099a941/e38e97a2735ffa8c/ ## Summary of changes Use pageserver API to learn about layers, instead of inspecting local disk, so that we will always agree with the pageserver about which layer are local. --- .../regress/test_pageserver_generations.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/test_runner/regress/test_pageserver_generations.py b/test_runner/regress/test_pageserver_generations.py index 994d09665c..78ef6b9165 100644 --- a/test_runner/regress/test_pageserver_generations.py +++ b/test_runner/regress/test_pageserver_generations.py @@ -486,16 +486,20 @@ def test_emergency_mode(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin): def evict_all_layers(env: NeonEnv, tenant_id: TenantId, timeline_id: TimelineId): - timeline_path = env.pageserver.timeline_dir(tenant_id, timeline_id) - initial_local_layers = sorted( - list(filter(lambda path: path.name != "metadata", timeline_path.glob("*"))) - ) client = env.pageserver.http_client() - for layer in initial_local_layers: - if "ephemeral" in layer.name or "temp" in layer.name: + + layer_map = client.layer_map_info(tenant_id, timeline_id) + + for layer in layer_map.historic_layers: + if layer.remote: + log.info( + f"Skipping trying to evict remote layer {tenant_id}/{timeline_id} {layer.layer_file_name}" + ) continue - log.info(f"Evicting layer {tenant_id}/{timeline_id} {layer.name}") - client.evict_layer(tenant_id=tenant_id, timeline_id=timeline_id, layer_name=layer.name) + log.info(f"Evicting layer {tenant_id}/{timeline_id} {layer.layer_file_name}") + client.evict_layer( + tenant_id=tenant_id, timeline_id=timeline_id, layer_name=layer.layer_file_name + ) def test_eviction_across_generations(neon_env_builder: NeonEnvBuilder): From c5949e1fd6de8a98ef70442c416db1a69cfd44f1 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Wed, 25 Oct 2023 14:59:43 +0300 Subject: [PATCH 08/69] misc smaller improvements (#5527) - finally add an `#[instrument]` to Timeline::create_image_layers, making it easier to see that something is happening because we create image layers - format some macro context code - add a warning not to create new validation functions a la parse do not validate Split off from #5198. --- pageserver/src/lib.rs | 4 ++++ pageserver/src/tenant.rs | 4 ++-- pageserver/src/tenant/timeline.rs | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pageserver/src/lib.rs b/pageserver/src/lib.rs index 32d43053bd..e71130e8af 100644 --- a/pageserver/src/lib.rs +++ b/pageserver/src/lib.rs @@ -149,6 +149,10 @@ fn ends_with_suffix(path: &Utf8Path, suffix: &str) -> bool { } } +// FIXME: DO NOT ADD new query methods like this, which will have a next step of parsing timelineid +// from the directory name. Instead create type "UninitMark(TimelineId)" and only parse it once +// from the name. + pub fn is_uninit_mark(path: &Utf8Path) -> bool { ends_with_suffix(path, TIMELINE_UNINIT_MARK_SUFFIX) } diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 82f82dcd09..f8895c32dc 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -1057,8 +1057,8 @@ impl Tenant { TimelineId::try_from(timeline_uninit_mark_file.file_stem()) .with_context(|| { format!( - "Could not parse timeline id out of the timeline uninit mark name {timeline_uninit_mark_file}", - ) + "Could not parse timeline id out of the timeline uninit mark name {timeline_uninit_mark_file}", + ) })?; let timeline_dir = self.conf.timeline_path(&self.tenant_id, &timeline_id); if let Err(e) = diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 8ce6aa2108..201541244a 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -3079,6 +3079,7 @@ impl Timeline { Ok(false) } + #[tracing::instrument(skip_all, fields(%lsn, %force))] async fn create_image_layers( &self, partitioning: &KeyPartitioning, From 4ae2d1390d1a0f7ba9caf1507eddf396e4d5d752 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Wed, 25 Oct 2023 15:01:19 +0300 Subject: [PATCH 09/69] refactor(remote_timeline_client): Split deletion into unlinking + deletion (#5645) Quest: #4745. Prerequisite for #4938. Original https://github.com/neondatabase/neon/pull/4938#issuecomment-1777150665. The new Layer implementation has so far been using `RemoteTimelineClient::schedule_layer_file_deletion` from `Layer::drop` but it was noticed that this could mean that the L0s compaction wanted to remove could linger in the index part for longer time or be left there for longer time. Solution is to split the `RemoteTimelineClient::schedule_layer_file_deletion` into two parts: - unlinking from index_part.json, to be called from end of compaction and gc - scheduling of actual deletions, to be called from `Layer::drop` The added methods are added unused. --- .../src/tenant/remote_timeline_client.rs | 150 ++++++++++++------ 1 file changed, 99 insertions(+), 51 deletions(-) diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 06e2292156..cad86fb4aa 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -653,11 +653,10 @@ impl RemoteTimelineClient { /// Launch a delete operation in the background. /// - /// The operation does not modify local state but assumes the local files have already been - /// deleted, and is used to mirror those changes to remote. + /// The operation does not modify local filesystem state. /// /// Note: This schedules an index file upload before the deletions. The - /// deletion won't actually be performed, until any previously scheduled + /// deletion won't actually be performed, until all previously scheduled /// upload operations, and the index file upload, have completed /// successfully. pub fn schedule_layer_file_deletion( @@ -667,56 +666,11 @@ impl RemoteTimelineClient { let mut guard = self.upload_queue.lock().unwrap(); let upload_queue = guard.initialized_mut()?; - // Deleting layers doesn't affect the values stored in TimelineMetadata, - // so we don't need update it. Just serialize it. - let metadata = upload_queue.latest_metadata.clone(); - - // Update the remote index file, removing the to-be-deleted files from the index, - // before deleting the actual files. - // - // Once we start removing files from upload_queue.latest_files, there's - // no going back! Otherwise, some of the files would already be removed - // from latest_files, but not yet scheduled for deletion. Use a closure - // to syntactically forbid ? or bail! calls here. let no_bail_here = || { - // Decorate our list of names with each name's generation, dropping - // makes that are unexpectedly missing from our metadata. - let with_generations: Vec<_> = names - .into_iter() - .filter_map(|name| { - // Remove from latest_files, learning the file's remote generation in the process - let meta = upload_queue.latest_files.remove(&name); + let with_generations = + self.schedule_unlinking_of_layers_from_index_part0(upload_queue, names); - if let Some(meta) = meta { - upload_queue.latest_files_changes_since_metadata_upload_scheduled += 1; - Some((name, meta.generation)) - } else { - // This can only happen if we forgot to to schedule the file upload - // before scheduling the delete. Log it because it is a rare/strange - // situation, and in case something is misbehaving, we'd like to know which - // layers experienced this. - info!( - "Deleting layer {name} not found in latest_files list, never uploaded?" - ); - None - } - }) - .collect(); - - if upload_queue.latest_files_changes_since_metadata_upload_scheduled > 0 { - self.schedule_index_upload(upload_queue, metadata); - } - - for (name, gen) in &with_generations { - info!("scheduling deletion of layer {}{}", name, gen.get_suffix()); - } - - // schedule the actual deletions - let op = UploadOp::Delete(Delete { - layers: with_generations, - }); - self.calls_unfinished_metric_begin(&op); - upload_queue.queued_operations.push_back(op); + self.schedule_deletion_of_unlinked0(upload_queue, with_generations); // Launch the tasks immediately, if possible self.launch_queued_tasks(upload_queue); @@ -725,6 +679,100 @@ impl RemoteTimelineClient { Ok(()) } + /// Unlinks the layer files from `index_part.json` but does not yet schedule deletion for the + /// layer files, leaving them dangling. + /// + /// The files will be leaked in remote storage unless [`Self::schedule_deletion_of_unlinked`] + /// is invoked on them. + #[allow(unused)] // will be used by PR#4938 + pub(crate) fn schedule_unlinking_of_layers_from_index_part( + self: &Arc, + names: Vec, + ) -> anyhow::Result<()> { + let mut guard = self.upload_queue.lock().unwrap(); + let upload_queue = guard.initialized_mut()?; + + // just forget the return value; after uploading the next index_part.json, we can consider + // the layer files as "dangling". this is fine however. + self.schedule_unlinking_of_layers_from_index_part0(upload_queue, names); + + self.launch_queued_tasks(upload_queue); + + Ok(()) + } + + /// Update the remote index file, removing the to-be-deleted files from the index, + /// allowing scheduling of actual deletions later. + fn schedule_unlinking_of_layers_from_index_part0( + self: &Arc, + upload_queue: &mut UploadQueueInitialized, + names: Vec, + ) -> Vec<(LayerFileName, Generation)> { + // Deleting layers doesn't affect the values stored in TimelineMetadata, + // so we don't need update it. Just serialize it. + let metadata = upload_queue.latest_metadata.clone(); + + // Decorate our list of names with each name's generation, dropping + // makes that are unexpectedly missing from our metadata. + let with_generations: Vec<_> = names + .into_iter() + .filter_map(|name| { + // Remove from latest_files, learning the file's remote generation in the process + let meta = upload_queue.latest_files.remove(&name); + + if let Some(meta) = meta { + upload_queue.latest_files_changes_since_metadata_upload_scheduled += 1; + Some((name, meta.generation)) + } else { + // This can only happen if we forgot to to schedule the file upload + // before scheduling the delete. Log it because it is a rare/strange + // situation, and in case something is misbehaving, we'd like to know which + // layers experienced this. + info!("Deleting layer {name} not found in latest_files list, never uploaded?"); + None + } + }) + .collect(); + + if upload_queue.latest_files_changes_since_metadata_upload_scheduled > 0 { + self.schedule_index_upload(upload_queue, metadata); + } + + with_generations + } + + /// Schedules deletion for layer files which have previously been unlinked from the + /// `index_part.json`. + #[allow(unused)] // will be used by Layer::drop in PR#4938 + pub(crate) fn schedule_deletion_of_unlinked( + self: &Arc, + layers: Vec<(LayerFileName, Generation)>, + ) -> anyhow::Result<()> { + let mut guard = self.upload_queue.lock().unwrap(); + let upload_queue = guard.initialized_mut()?; + + self.schedule_deletion_of_unlinked0(upload_queue, layers); + self.launch_queued_tasks(upload_queue); + Ok(()) + } + + fn schedule_deletion_of_unlinked0( + self: &Arc, + upload_queue: &mut UploadQueueInitialized, + with_generations: Vec<(LayerFileName, Generation)>, + ) { + for (name, gen) in &with_generations { + info!("scheduling deletion of layer {}{}", name, gen.get_suffix()); + } + + // schedule the actual deletions + let op = UploadOp::Delete(Delete { + layers: with_generations, + }); + self.calls_unfinished_metric_begin(&op); + upload_queue.queued_operations.push_back(op); + } + /// /// Wait for all previously scheduled uploads/deletions to complete /// From a461c459d8ce3e6588e59f7e6d37cef2131cc38d Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Wed, 25 Oct 2023 13:20:45 +0100 Subject: [PATCH 10/69] fix http pool test (#5653) ## Problem We defer the returning of connections the the connection pool. It's possible for our test to be faster than the returning of connections - which then gets a differing process ID because it opens a new connection. ## Summary of changes 1. Delay the tests just a little (20ms) to give more chance for connections to return. 2. Correlate connection IDs with the connection logs a bit more --- proxy/src/http/conn_pool.rs | 64 ++++++++++++++++++++++++------- test_runner/regress/test_proxy.py | 10 +++++ 2 files changed, 61 insertions(+), 13 deletions(-) diff --git a/proxy/src/http/conn_pool.rs b/proxy/src/http/conn_pool.rs index 7a5f00d627..5218a44479 100644 --- a/proxy/src/http/conn_pool.rs +++ b/proxy/src/http/conn_pool.rs @@ -191,22 +191,39 @@ impl GlobalConnPool { // ok return cached connection if found and establish a new one otherwise let new_client = if let Some(client) = client { if client.inner.is_closed() { - info!("pool: cached connection '{conn_info}' is closed, opening a new one"); - connect_to_compute(self.proxy_config, conn_info, session_id, latency_timer).await + let conn_id = uuid::Uuid::new_v4(); + info!(%conn_id, "pool: cached connection '{conn_info}' is closed, opening a new one"); + connect_to_compute( + self.proxy_config, + conn_info, + conn_id, + session_id, + latency_timer, + ) + .await } else { info!("pool: reusing connection '{conn_info}'"); client.session.send(session_id)?; latency_timer.pool_hit(); latency_timer.success(); return Ok(Client { + conn_id: client.conn_id, inner: Some(client), span: Span::current(), pool, }); } } else { - info!("pool: opening a new connection '{conn_info}'"); - connect_to_compute(self.proxy_config, conn_info, session_id, latency_timer).await + let conn_id = uuid::Uuid::new_v4(); + info!(%conn_id, "pool: opening a new connection '{conn_info}'"); + connect_to_compute( + self.proxy_config, + conn_info, + conn_id, + session_id, + latency_timer, + ) + .await }; match &new_client { @@ -243,6 +260,7 @@ impl GlobalConnPool { } new_client.map(|inner| Client { + conn_id: inner.conn_id, inner: Some(inner), span: Span::current(), pool, @@ -250,16 +268,18 @@ impl GlobalConnPool { } fn put(&self, conn_info: &ConnInfo, client: ClientInner) -> anyhow::Result<()> { + let conn_id = client.conn_id; + // We want to hold this open while we return. This ensures that the pool can't close // while we are in the middle of returning the connection. let closed = self.closed.read(); if *closed { - info!("pool: throwing away connection '{conn_info}' because pool is closed"); + info!(%conn_id, "pool: throwing away connection '{conn_info}' because pool is closed"); return Ok(()); } if client.inner.is_closed() { - info!("pool: throwing away connection '{conn_info}' because connection is closed"); + info!(%conn_id, "pool: throwing away connection '{conn_info}' because connection is closed"); return Ok(()); } @@ -291,9 +311,9 @@ impl GlobalConnPool { // do logging outside of the mutex if returned { - info!("pool: returning connection '{conn_info}' back to the pool, total_conns={total_conns}, for this (db, user)={per_db_size}"); + info!(%conn_id, "pool: returning connection '{conn_info}' back to the pool, total_conns={total_conns}, for this (db, user)={per_db_size}"); } else { - info!("pool: throwing away connection '{conn_info}' because pool is full, total_conns={total_conns}"); + info!(%conn_id, "pool: throwing away connection '{conn_info}' because pool is full, total_conns={total_conns}"); } Ok(()) @@ -340,6 +360,7 @@ impl GlobalConnPool { struct TokioMechanism<'a> { conn_info: &'a ConnInfo, session_id: uuid::Uuid, + conn_id: uuid::Uuid, } #[async_trait] @@ -353,7 +374,14 @@ impl ConnectMechanism for TokioMechanism<'_> { node_info: &console::CachedNodeInfo, timeout: time::Duration, ) -> Result { - connect_to_compute_once(node_info, self.conn_info, timeout, self.session_id).await + connect_to_compute_once( + node_info, + self.conn_info, + timeout, + self.conn_id, + self.session_id, + ) + .await } fn update_connect_config(&self, _config: &mut compute::ConnCfg) {} @@ -366,6 +394,7 @@ impl ConnectMechanism for TokioMechanism<'_> { async fn connect_to_compute( config: &config::ProxyConfig, conn_info: &ConnInfo, + conn_id: uuid::Uuid, session_id: uuid::Uuid, latency_timer: LatencyTimer, ) -> anyhow::Result { @@ -401,6 +430,7 @@ async fn connect_to_compute( crate::proxy::connect_to_compute( &TokioMechanism { + conn_id, conn_info, session_id, }, @@ -416,6 +446,7 @@ async fn connect_to_compute_once( node_info: &console::CachedNodeInfo, conn_info: &ConnInfo, timeout: time::Duration, + conn_id: uuid::Uuid, mut session: uuid::Uuid, ) -> Result { let mut config = (*node_info.config).clone(); @@ -430,7 +461,6 @@ async fn connect_to_compute_once( let (tx, mut rx) = tokio::sync::watch::channel(session); - let conn_id = uuid::Uuid::new_v4(); let span = info_span!(parent: None, "connection", %conn_id); span.in_scope(|| { info!(%conn_info, %session, "new connection"); @@ -484,6 +514,7 @@ async fn connect_to_compute_once( inner: client, session: tx, ids, + conn_id, }) } @@ -491,6 +522,7 @@ struct ClientInner { inner: tokio_postgres::Client, session: tokio::sync::watch::Sender, ids: Ids, + conn_id: uuid::Uuid, } impl Client { @@ -500,12 +532,14 @@ impl Client { } pub struct Client { + conn_id: uuid::Uuid, span: Span, inner: Option, pool: Option<(ConnInfo, Arc)>, } pub struct Discard<'a> { + conn_id: uuid::Uuid, pool: &'a mut Option<(ConnInfo, Arc)>, } @@ -514,6 +548,7 @@ impl Client { let Self { inner, pool, + conn_id, span: _, } = self; ( @@ -521,7 +556,10 @@ impl Client { .as_mut() .expect("client inner should not be removed") .inner, - Discard { pool }, + Discard { + pool, + conn_id: *conn_id, + }, ) } @@ -537,13 +575,13 @@ impl Discard<'_> { pub fn check_idle(&mut self, status: ReadyForQueryStatus) { if status != ReadyForQueryStatus::Idle { if let Some((conn_info, _)) = self.pool.take() { - info!("pool: throwing away connection '{conn_info}' because connection is not idle") + info!(conn_id = %self.conn_id, "pool: throwing away connection '{conn_info}' because connection is not idle") } } } pub fn discard(&mut self) { if let Some((conn_info, _)) = self.pool.take() { - info!("pool: throwing away connection '{conn_info}' because connection is potentially in a broken state") + info!(conn_id = %self.conn_id, "pool: throwing away connection '{conn_info}' because connection is potentially in a broken state") } } } diff --git a/test_runner/regress/test_proxy.py b/test_runner/regress/test_proxy.py index a33b29549c..6388e9f365 100644 --- a/test_runner/regress/test_proxy.py +++ b/test_runner/regress/test_proxy.py @@ -1,5 +1,6 @@ import json import subprocess +import time from typing import Any, List, Optional, Tuple import psycopg2 @@ -364,10 +365,14 @@ def test_sql_over_http_pool(static_proxy: NeonProxy): pid1 = get_pid(200, "http")["rows"][0]["pid"] + time.sleep(0.02) + # query should be on the same connection rows = get_pid(200, "http")["rows"] assert rows == [{"pid": pid1}] + time.sleep(0.02) + # incorrect password should not work res = get_pid(400, "foobar") assert "password authentication failed for user" in res["message"] @@ -378,10 +383,14 @@ def test_sql_over_http_pool(static_proxy: NeonProxy): pid2 = get_pid(200, "http2")["rows"][0]["pid"] assert pid1 != pid2 + time.sleep(0.02) + # query should be on an existing connection pid = get_pid(200, "http2")["rows"][0]["pid"] assert pid in [pid1, pid2] + time.sleep(0.02) + # old password should not work res = get_pid(400, "http") assert "password authentication failed for user" in res["message"] @@ -419,6 +428,7 @@ def test_sql_over_http_pool_idle(static_proxy: NeonProxy): ) pid1 = query(200, GET_CONNECTION_PID_QUERY)["rows"][0]["pid"] + time.sleep(0.02) query(200, "BEGIN") pid2 = query(200, GET_CONNECTION_PID_QUERY)["rows"][0]["pid"] assert pid1 != pid2 From 8b8be7bed4184be684ff1a729fe90ba350a10da2 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 25 Oct 2023 13:29:30 +0100 Subject: [PATCH 11/69] tests: don't fail tests on torn log lines (#5655) ## Problem Tests that force-kill and restart a service can generate torn log lines that might match WARN|ERROR, but not match the allow expression that a test has loaded, e.g. https://neon-github-public-dev.s3.amazonaws.com/reports/pr-5651/6638398772/index.html#suites/7538959189f4501983ddd9e167836c8b/d272ba8a73e6945c ## Summary of changes Ignore log lines which match a regex for torn log lines on restart: they have two timestamps and the second line is an "INFO version"... message. --- test_runner/fixtures/neon_fixtures.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 38e17985ac..1a7db18b07 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1719,6 +1719,11 @@ class NeonPageserver(PgProtocol): break if error_or_warn.search(line): + # Is this a torn log line? This happens when force-killing a process and restarting + # Example: "2023-10-25T09:38:31.752314Z WARN deletion executo2023-10-25T09:38:31.875947Z INFO version: git-env:0f9452f76e8ccdfc88291bccb3f53e3016f40192" + if re.match("\\d{4}-\\d{2}-\\d{2}T.+\\d{4}-\\d{2}-\\d{2}T.+INFO version.+", line): + continue + # It's an ERROR or WARN. Is it in the allow-list? for a in self.allowed_errors: if re.match(a, line): From 4778b6a12e86075bc27c92cdb0a65de0dbe153d5 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Wed, 25 Oct 2023 14:25:13 +0100 Subject: [PATCH 12/69] Switch to querying new tests results DB (#5616) ## Problem We started to store test results in a new format in https://github.com/neondatabase/neon/pull/4549. This PR switches scripts to query this db. (we can completely remove old DB/ingestions scripts in a couple of weeks after the PR merged) ## Summary of changes - `scripts/benchmark_durations.py` query new database - `scripts/flaky_tests.py` query new database --- .../actions/allure-report-generate/action.yml | 4 + .github/workflows/build_and_test.yml | 4 +- scripts/benchmark_durations.py | 150 ++++++++---------- scripts/flaky_tests.py | 35 ++-- 4 files changed, 87 insertions(+), 106 deletions(-) diff --git a/.github/actions/allure-report-generate/action.yml b/.github/actions/allure-report-generate/action.yml index 14e1653cb0..abdbba802e 100644 --- a/.github/actions/allure-report-generate/action.yml +++ b/.github/actions/allure-report-generate/action.yml @@ -203,6 +203,10 @@ runs: COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} BASE_S3_URL: ${{ steps.generate-report.outputs.base-s3-url }} run: | + if [ ! -d "${WORKDIR}/report/data/test-cases" ]; then + exit 0 + fi + export DATABASE_URL=${REGRESS_TEST_RESULT_CONNSTR_NEW} ./scripts/pysync diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 9a3d352ad0..58ddc66d5d 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -433,7 +433,7 @@ jobs: rerun_flaky: true pg_version: ${{ matrix.pg_version }} env: - TEST_RESULT_CONNSTR: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR }} + TEST_RESULT_CONNSTR: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }} CHECK_ONDISK_DATA_COMPATIBILITY: nonempty - name: Merge and upload coverage data @@ -468,7 +468,7 @@ jobs: env: VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" - TEST_RESULT_CONNSTR: "${{ secrets.REGRESS_TEST_RESULT_CONNSTR }}" + TEST_RESULT_CONNSTR: "${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }}" # XXX: no coverage data handling here, since benchmarks are run on release builds, # while coverage is currently collected for the debug ones diff --git a/scripts/benchmark_durations.py b/scripts/benchmark_durations.py index 37f8470038..781d0d2d1e 100755 --- a/scripts/benchmark_durations.py +++ b/scripts/benchmark_durations.py @@ -15,28 +15,15 @@ The script fetches the durations of benchmarks from the database and stores it i BENCHMARKS_DURATION_QUERY = """ SELECT - DISTINCT parent_suite, suite, test, - PERCENTILE_DISC(%s) WITHIN GROUP (ORDER BY duration_ms) as percentile_ms - FROM - ( - SELECT - jsonb_array_elements(data -> 'children') ->> 'name' as parent_suite, - jsonb_array_elements(jsonb_array_elements(data -> 'children') -> 'children') ->> 'name' as suite, - jsonb_array_elements(jsonb_array_elements(jsonb_array_elements(data -> 'children') -> 'children') -> 'children') ->> 'name' as test, - jsonb_array_elements(jsonb_array_elements(jsonb_array_elements(data -> 'children') -> 'children') -> 'children') ->> 'status' as status, - to_timestamp((jsonb_array_elements(jsonb_array_elements(jsonb_array_elements(data -> 'children') -> 'children') -> 'children') -> 'time' -> 'start')::bigint / 1000)::date as timestamp, - (jsonb_array_elements(jsonb_array_elements(jsonb_array_elements(data -> 'children') -> 'children') -> 'children') -> 'time' -> 'duration')::int as duration_ms - FROM - regress_test_results - WHERE - reference = 'refs/heads/main' - ) data + DISTINCT parent_suite, suite, name, + PERCENTILE_DISC(%s) WITHIN GROUP (ORDER BY duration) as percentile_ms + FROM results WHERE - timestamp > CURRENT_DATE - INTERVAL '%s' day + started_at > CURRENT_DATE - INTERVAL '%s' day AND parent_suite = 'test_runner.performance' AND status = 'passed' GROUP BY - parent_suite, suite, test + parent_suite, suite, name ; """ @@ -44,68 +31,69 @@ BENCHMARKS_DURATION_QUERY = """ # the total duration varies from 8 to 40 minutes. # We use some pre-collected durations as a fallback to have a better distribution. FALLBACK_DURATION = { - "test_runner/performance/test_branch_creation.py::test_branch_creation_heavy_write[20]": 57.0, - "test_runner/performance/test_branch_creation.py::test_branch_creation_many_relations": 28.0, - "test_runner/performance/test_branch_creation.py::test_branch_creation_many[1024]": 71.0, - "test_runner/performance/test_branching.py::test_compare_child_and_root_pgbench_perf": 27.0, - "test_runner/performance/test_branching.py::test_compare_child_and_root_read_perf": 11.0, - "test_runner/performance/test_branching.py::test_compare_child_and_root_write_perf": 30.0, - "test_runner/performance/test_bulk_insert.py::test_bulk_insert[neon]": 40.0, - "test_runner/performance/test_bulk_insert.py::test_bulk_insert[vanilla]": 5.0, - "test_runner/performance/test_bulk_tenant_create.py::test_bulk_tenant_create[1]": 3.0, - "test_runner/performance/test_bulk_tenant_create.py::test_bulk_tenant_create[5]": 10.0, - "test_runner/performance/test_bulk_tenant_create.py::test_bulk_tenant_create[10]": 19.0, - "test_runner/performance/test_bulk_update.py::test_bulk_update[10]": 66.0, - "test_runner/performance/test_bulk_update.py::test_bulk_update[50]": 30.0, - "test_runner/performance/test_bulk_update.py::test_bulk_update[100]": 60.0, - "test_runner/performance/test_compaction.py::test_compaction": 77.0, - "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_ro_with_pgbench_select_only[neon-5-10-100]": 11.0, - "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_ro_with_pgbench_select_only[vanilla-5-10-100]": 16.0, - "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_rw_with_pgbench_default[neon-5-10-100]": 11.0, - "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_rw_with_pgbench_default[vanilla-5-10-100]": 18.0, - "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_wal_with_pgbench_default[neon-5-10-100]": 11.0, - "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_wal_with_pgbench_default[vanilla-5-10-100]": 16.0, - "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_wo_with_heavy_write[neon-10-1]": 11.0, - "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_wo_with_heavy_write[neon-10-10]": 11.0, - "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_wo_with_heavy_write[vanilla-10-1]": 10.0, - "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_wo_with_heavy_write[vanilla-10-10]": 10.0, - "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_wo_with_pgbench_simple_update[neon-5-10-100]": 11.0, - "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_wo_with_pgbench_simple_update[vanilla-5-10-100]": 16.0, - "test_runner/performance/test_copy.py::test_copy[neon]": 12.0, - "test_runner/performance/test_copy.py::test_copy[vanilla]": 10.0, - "test_runner/performance/test_gc_feedback.py::test_gc_feedback": 284.0, - "test_runner/performance/test_gist_build.py::test_gist_buffering_build[neon]": 11.0, - "test_runner/performance/test_gist_build.py::test_gist_buffering_build[vanilla]": 7.0, - "test_runner/performance/test_latency.py::test_measure_read_latency_heavy_write_workload[neon-1]": 85.0, - "test_runner/performance/test_latency.py::test_measure_read_latency_heavy_write_workload[vanilla-1]": 29.0, - "test_runner/performance/test_layer_map.py::test_layer_map": 44.0, - "test_runner/performance/test_parallel_copy_to.py::test_parallel_copy_different_tables[neon]": 16.0, - "test_runner/performance/test_parallel_copy_to.py::test_parallel_copy_different_tables[vanilla]": 67.0, - "test_runner/performance/test_parallel_copy_to.py::test_parallel_copy_same_table[neon]": 67.0, - "test_runner/performance/test_parallel_copy_to.py::test_parallel_copy_same_table[vanilla]": 80.0, - "test_runner/performance/test_perf_pgbench.py::test_pgbench[neon-45-10]": 102.0, - "test_runner/performance/test_perf_pgbench.py::test_pgbench[vanilla-45-10]": 99.0, - "test_runner/performance/test_random_writes.py::test_random_writes[neon]": 9.0, - "test_runner/performance/test_random_writes.py::test_random_writes[vanilla]": 2.0, - "test_runner/performance/test_seqscans.py::test_seqscans[neon-100000-100-0]": 4.0, - "test_runner/performance/test_seqscans.py::test_seqscans[neon-10000000-1-0]": 80.0, - "test_runner/performance/test_seqscans.py::test_seqscans[neon-10000000-1-4]": 68.0, - "test_runner/performance/test_seqscans.py::test_seqscans[vanilla-100000-100-0]": 0.0, - "test_runner/performance/test_seqscans.py::test_seqscans[vanilla-10000000-1-0]": 11.0, - "test_runner/performance/test_seqscans.py::test_seqscans[vanilla-10000000-1-4]": 10.0, - "test_runner/performance/test_startup.py::test_startup_simple": 2.0, - "test_runner/performance/test_startup.py::test_startup": 539.0, - "test_runner/performance/test_wal_backpressure.py::test_heavy_write_workload[neon_off-10-5-5]": 375.0, - "test_runner/performance/test_wal_backpressure.py::test_heavy_write_workload[neon_on-10-5-5]": 370.0, - "test_runner/performance/test_wal_backpressure.py::test_heavy_write_workload[vanilla-10-5-5]": 94.0, - "test_runner/performance/test_wal_backpressure.py::test_pgbench_intensive_init_workload[neon_off-1000]": 164.0, - "test_runner/performance/test_wal_backpressure.py::test_pgbench_intensive_init_workload[neon_on-1000]": 274.0, - "test_runner/performance/test_wal_backpressure.py::test_pgbench_intensive_init_workload[vanilla-1000]": 949.0, - "test_runner/performance/test_wal_backpressure.py::test_pgbench_simple_update_workload[neon_off-45-100]": 142.0, - "test_runner/performance/test_wal_backpressure.py::test_pgbench_simple_update_workload[neon_on-45-100]": 151.0, - "test_runner/performance/test_wal_backpressure.py::test_pgbench_simple_update_workload[vanilla-45-100]": 182.0, - "test_runner/performance/test_write_amplification.py::test_write_amplification[neon]": 13.0, - "test_runner/performance/test_write_amplification.py::test_write_amplification[vanilla]": 16.0, + "test_runner/performance/test_branch_creation.py::test_branch_creation_heavy_write[20]": 62.144, + "test_runner/performance/test_branch_creation.py::test_branch_creation_many[1024]": 90.941, + "test_runner/performance/test_branch_creation.py::test_branch_creation_many_relations": 26.053, + "test_runner/performance/test_branching.py::test_compare_child_and_root_pgbench_perf": 25.67, + "test_runner/performance/test_branching.py::test_compare_child_and_root_read_perf": 14.497, + "test_runner/performance/test_branching.py::test_compare_child_and_root_write_perf": 18.852, + "test_runner/performance/test_bulk_insert.py::test_bulk_insert[neon]": 26.572, + "test_runner/performance/test_bulk_insert.py::test_bulk_insert[vanilla]": 6.259, + "test_runner/performance/test_bulk_tenant_create.py::test_bulk_tenant_create[10]": 21.206, + "test_runner/performance/test_bulk_tenant_create.py::test_bulk_tenant_create[1]": 3.474, + "test_runner/performance/test_bulk_tenant_create.py::test_bulk_tenant_create[5]": 11.262, + "test_runner/performance/test_bulk_update.py::test_bulk_update[100]": 94.225, + "test_runner/performance/test_bulk_update.py::test_bulk_update[10]": 68.159, + "test_runner/performance/test_bulk_update.py::test_bulk_update[50]": 76.719, + "test_runner/performance/test_compaction.py::test_compaction": 110.222, + "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_ro_with_pgbench_select_only[neon-5-10-100]": 10.743, + "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_ro_with_pgbench_select_only[vanilla-5-10-100]": 16.541, + "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_rw_with_pgbench_default[neon-5-10-100]": 11.109, + "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_rw_with_pgbench_default[vanilla-5-10-100]": 18.121, + "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_wal_with_pgbench_default[neon-5-10-100]": 11.3, + "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_wal_with_pgbench_default[vanilla-5-10-100]": 16.086, + "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_wo_with_heavy_write[neon-10-10]": 12.024, + "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_wo_with_heavy_write[neon-10-1]": 11.14, + "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_wo_with_heavy_write[vanilla-10-10]": 10.375, + "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_wo_with_heavy_write[vanilla-10-1]": 10.075, + "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_wo_with_pgbench_simple_update[neon-5-10-100]": 11.147, + "test_runner/performance/test_compare_pg_stats.py::test_compare_pg_stats_wo_with_pgbench_simple_update[vanilla-5-10-100]": 16.321, + "test_runner/performance/test_copy.py::test_copy[neon]": 16.579, + "test_runner/performance/test_copy.py::test_copy[vanilla]": 10.094, + "test_runner/performance/test_gc_feedback.py::test_gc_feedback": 590.157, + "test_runner/performance/test_gist_build.py::test_gist_buffering_build[neon]": 14.102, + "test_runner/performance/test_gist_build.py::test_gist_buffering_build[vanilla]": 8.677, + "test_runner/performance/test_latency.py::test_measure_read_latency_heavy_write_workload[neon-1]": 31.079, + "test_runner/performance/test_latency.py::test_measure_read_latency_heavy_write_workload[vanilla-1]": 38.119, + "test_runner/performance/test_layer_map.py::test_layer_map": 24.784, + "test_runner/performance/test_logical_replication.py::test_logical_replication": 117.707, + "test_runner/performance/test_parallel_copy_to.py::test_parallel_copy_different_tables[neon]": 21.194, + "test_runner/performance/test_parallel_copy_to.py::test_parallel_copy_different_tables[vanilla]": 59.068, + "test_runner/performance/test_parallel_copy_to.py::test_parallel_copy_same_table[neon]": 73.235, + "test_runner/performance/test_parallel_copy_to.py::test_parallel_copy_same_table[vanilla]": 82.586, + "test_runner/performance/test_perf_pgbench.py::test_pgbench[neon-45-10]": 106.536, + "test_runner/performance/test_perf_pgbench.py::test_pgbench[vanilla-45-10]": 98.753, + "test_runner/performance/test_random_writes.py::test_random_writes[neon]": 6.975, + "test_runner/performance/test_random_writes.py::test_random_writes[vanilla]": 3.69, + "test_runner/performance/test_seqscans.py::test_seqscans[neon-100000-100-0]": 3.529, + "test_runner/performance/test_seqscans.py::test_seqscans[neon-10000000-1-0]": 64.522, + "test_runner/performance/test_seqscans.py::test_seqscans[neon-10000000-1-4]": 40.964, + "test_runner/performance/test_seqscans.py::test_seqscans[vanilla-100000-100-0]": 0.55, + "test_runner/performance/test_seqscans.py::test_seqscans[vanilla-10000000-1-0]": 12.189, + "test_runner/performance/test_seqscans.py::test_seqscans[vanilla-10000000-1-4]": 13.899, + "test_runner/performance/test_startup.py::test_startup": 890.114, + "test_runner/performance/test_startup.py::test_startup_simple": 2.51, + "test_runner/performance/test_wal_backpressure.py::test_heavy_write_workload[neon_off-10-5-5]": 527.245, + "test_runner/performance/test_wal_backpressure.py::test_heavy_write_workload[neon_on-10-5-5]": 583.46, + "test_runner/performance/test_wal_backpressure.py::test_heavy_write_workload[vanilla-10-5-5]": 113.653, + "test_runner/performance/test_wal_backpressure.py::test_pgbench_intensive_init_workload[neon_off-1000]": 233.728, + "test_runner/performance/test_wal_backpressure.py::test_pgbench_intensive_init_workload[neon_on-1000]": 419.093, + "test_runner/performance/test_wal_backpressure.py::test_pgbench_intensive_init_workload[vanilla-1000]": 982.461, + "test_runner/performance/test_wal_backpressure.py::test_pgbench_simple_update_workload[neon_off-45-100]": 116.522, + "test_runner/performance/test_wal_backpressure.py::test_pgbench_simple_update_workload[neon_on-45-100]": 115.583, + "test_runner/performance/test_wal_backpressure.py::test_pgbench_simple_update_workload[vanilla-45-100]": 155.282, + "test_runner/performance/test_write_amplification.py::test_write_amplification[neon]": 26.704, + "test_runner/performance/test_write_amplification.py::test_write_amplification[vanilla]": 16.088, } @@ -130,7 +118,7 @@ def main(args: argparse.Namespace): res = FALLBACK_DURATION for row in rows: - pytest_name = f"{row['parent_suite'].replace('.', '/')}/{row['suite']}.py::{row['test']}" + pytest_name = f"{row['parent_suite'].replace('.', '/')}/{row['suite']}.py::{row['name']}" duration = row["percentile_ms"] / 1000 logging.info(f"\t{pytest_name}: {duration}") res[pytest_name] = duration diff --git a/scripts/flaky_tests.py b/scripts/flaky_tests.py index 5ef2f76798..b07e4bea9b 100755 --- a/scripts/flaky_tests.py +++ b/scripts/flaky_tests.py @@ -9,28 +9,15 @@ from typing import DefaultDict, Dict import psycopg2 import psycopg2.extras -# We call the test "flaky" if it failed at least once on the main branch in the last N=10 days. FLAKY_TESTS_QUERY = """ SELECT - DISTINCT parent_suite, suite, REGEXP_REPLACE(test, '(release|debug)-pg(\\d+)-?', '') as deparametrized_test - FROM - ( - SELECT - reference, - jsonb_array_elements(data -> 'children') ->> 'name' as parent_suite, - jsonb_array_elements(jsonb_array_elements(data -> 'children') -> 'children') ->> 'name' as suite, - jsonb_array_elements(jsonb_array_elements(jsonb_array_elements(data -> 'children') -> 'children') -> 'children') ->> 'name' as test, - jsonb_array_elements(jsonb_array_elements(jsonb_array_elements(data -> 'children') -> 'children') -> 'children') ->> 'status' as status, - jsonb_array_elements(jsonb_array_elements(jsonb_array_elements(data -> 'children') -> 'children') -> 'children') ->> 'retriesStatusChange' as retries_status_change, - to_timestamp((jsonb_array_elements(jsonb_array_elements(jsonb_array_elements(data -> 'children') -> 'children') -> 'children') -> 'time' ->> 'start')::bigint / 1000)::date as timestamp - FROM - regress_test_results - ) data + DISTINCT parent_suite, suite, name + FROM results WHERE - timestamp > CURRENT_DATE - INTERVAL '%s' day + started_at > CURRENT_DATE - INTERVAL '%s' day AND ( (status IN ('failed', 'broken') AND reference = 'refs/heads/main') - OR retries_status_change::boolean + OR flaky ) ; """ @@ -63,12 +50,14 @@ def main(args: argparse.Namespace): if row["parent_suite"] != "test_runner.regress": continue - deparametrized_test = row["deparametrized_test"] - dash_if_needed = "" if deparametrized_test.endswith("[]") else "-" - parametrized_test = deparametrized_test.replace( - "[", - f"[{build_type}-pg{pg_version}{dash_if_needed}", - ) + if row["name"].endswith("]"): + parametrized_test = row["name"].replace( + "[", + f"[{build_type}-pg{pg_version}-", + ) + else: + parametrized_test = f"{row['name']}[{build_type}-pg{pg_version}]" + res[row["parent_suite"]][row["suite"]][parametrized_test] = True logging.info( From 5683ae9eabddcf1d29c70dd45f30a219d194e9ee Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 25 Oct 2023 14:55:37 +0100 Subject: [PATCH 13/69] pageserver: suppress some of the most common spurious warnings (#5658) Two of the most common spurious log messages: - broker connections terminate & we log at error severity. Unfortunately tonic gives us an "Unknown" error so to suppress these we're doing string matching. It's hacky but worthwhile for operations. - the first iteration of tenant background tasks tends to over-run its schedule and emit a warning. Ultimately we should fix these to run on time, but for now we are not benefiting from polluting our logs with the warnings. --- pageserver/src/tenant/tasks.rs | 36 ++++++++++++------- .../walreceiver/connection_manager.rs | 16 ++++++--- storage_broker/src/lib.rs | 3 +- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/pageserver/src/tenant/tasks.rs b/pageserver/src/tenant/tasks.rs index 00c8ced68a..a845456181 100644 --- a/pageserver/src/tenant/tasks.rs +++ b/pageserver/src/tenant/tasks.rs @@ -159,11 +159,8 @@ async fn compaction_loop(tenant: Arc, cancel: CancellationToken) { // TODO: we shouldn't need to await to find tenant and this could be moved outside of // loop, #3501. There are also additional "allowed_errors" in tests. - if first { - first = false; - if random_init_delay(period, &cancel).await.is_err() { - break; - } + if first && random_init_delay(period, &cancel).await.is_err() { + break; } let started_at = Instant::now(); @@ -183,7 +180,16 @@ async fn compaction_loop(tenant: Arc, cancel: CancellationToken) { } }; - warn_when_period_overrun(started_at.elapsed(), period, BackgroundLoopKind::Compaction); + if !first { + // The first iteration is typically much slower, because all tenants compete for the + // compaction sempahore to run, and because of concurrent startup work like initializing + // logical sizes. To avoid routinely spamming warnings, we suppress this log on first iteration. + warn_when_period_overrun( + started_at.elapsed(), + period, + BackgroundLoopKind::Compaction, + ); + } // Sleep if tokio::time::timeout(sleep_duration, cancel.cancelled()) @@ -192,6 +198,8 @@ async fn compaction_loop(tenant: Arc, cancel: CancellationToken) { { break; } + + first = false; } } .await; @@ -223,11 +231,8 @@ async fn gc_loop(tenant: Arc, cancel: CancellationToken) { let period = tenant.get_gc_period(); - if first { - first = false; - if random_init_delay(period, &cancel).await.is_err() { - break; - } + if first && random_init_delay(period, &cancel).await.is_err() { + break; } let started_at = Instant::now(); @@ -251,7 +256,12 @@ async fn gc_loop(tenant: Arc, cancel: CancellationToken) { } }; - warn_when_period_overrun(started_at.elapsed(), period, BackgroundLoopKind::Gc); + if !first { + // The first iteration is typically much slower, because all tenants compete for the + // compaction sempahore to run, and because of concurrent startup work like initializing + // logical sizes. To avoid routinely spamming warnings, we suppress this log on first iteration. + warn_when_period_overrun(started_at.elapsed(), period, BackgroundLoopKind::Gc); + } // Sleep if tokio::time::timeout(sleep_duration, cancel.cancelled()) @@ -260,6 +270,8 @@ async fn gc_loop(tenant: Arc, cancel: CancellationToken) { { break; } + + first = false; } } .await; diff --git a/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs index 36960c631c..e28c4a5f69 100644 --- a/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs +++ b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs @@ -26,8 +26,7 @@ use storage_broker::proto::subscribe_safekeeper_info_request::SubscriptionKey; use storage_broker::proto::SafekeeperTimelineInfo; use storage_broker::proto::SubscribeSafekeeperInfoRequest; use storage_broker::proto::TenantTimelineId as ProtoTenantTimelineId; -use storage_broker::BrokerClientChannel; -use storage_broker::Streaming; +use storage_broker::{BrokerClientChannel, Code, Streaming}; use tokio::select; use tracing::*; @@ -137,8 +136,17 @@ pub(super) async fn connection_manager_loop_step( broker_update = broker_subscription.message() => { match broker_update { Ok(Some(broker_update)) => connection_manager_state.register_timeline_update(broker_update), - Err(e) => { - error!("broker subscription failed: {e}"); + Err(status) => { + match status.code() { + Code::Unknown if status.message().contains("stream closed because of a broken pipe") => { + // tonic's error handling doesn't provide a clear code for disconnections: we get + // "h2 protocol error: error reading a body from connection: stream closed because of a broken pipe" + info!("broker disconnected: {status}"); + }, + _ => { + warn!("broker subscription failed: {status}"); + } + } return ControlFlow::Continue(()); } Ok(None) => { diff --git a/storage_broker/src/lib.rs b/storage_broker/src/lib.rs index 3f6fa35cbe..aa5d0bad5f 100644 --- a/storage_broker/src/lib.rs +++ b/storage_broker/src/lib.rs @@ -4,7 +4,7 @@ use std::task::{Context, Poll}; use std::time::Duration; use tonic::codegen::StdError; use tonic::transport::{ClientTlsConfig, Endpoint}; -use tonic::{transport::Channel, Code, Status}; +use tonic::{transport::Channel, Status}; use utils::id::{TenantId, TenantTimelineId, TimelineId}; use proto::{ @@ -23,6 +23,7 @@ pub mod proto { pub mod metrics; // Re-exports to avoid direct tonic dependency in user crates. +pub use tonic::Code; pub use tonic::Request; pub use tonic::Streaming; From 32126d705b940cee5393eaf81142d3eb2e401a47 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Wed, 25 Oct 2023 15:43:03 +0100 Subject: [PATCH 14/69] proxy refactor serverless (#4685) ## Problem Our serverless backend was a bit jumbled. As a comment indicated, we were handling SQL-over-HTTP in our `websocket.rs` file. I've extracted out the `sql_over_http` and `websocket` files from the `http` module and put them into a new module called `serverless`. ## Summary of changes ```sh mkdir proxy/src/serverless mv proxy/src/http/{conn_pool,sql_over_http,websocket}.rs proxy/src/serverless/ mv proxy/src/http/server.rs proxy/src/http/health_server.rs mv proxy/src/metrics proxy/src/usage_metrics.rs ``` I have also extracted the hyper server and handler from websocket.rs into `serverless.rs` --- proxy/src/bin/proxy.rs | 21 +- proxy/src/http.rs | 5 +- .../src/http/{server.rs => health_server.rs} | 0 proxy/src/lib.rs | 3 +- proxy/src/proxy.rs | 2 +- .../src/{http/websocket.rs => serverless.rs} | 319 +++++------------- proxy/src/{http => serverless}/conn_pool.rs | 2 +- .../src/{http => serverless}/sql_over_http.rs | 0 proxy/src/serverless/websocket.rs | 146 ++++++++ proxy/src/{metrics.rs => usage_metrics.rs} | 0 10 files changed, 255 insertions(+), 243 deletions(-) rename proxy/src/http/{server.rs => health_server.rs} (100%) rename proxy/src/{http/websocket.rs => serverless.rs} (51%) rename proxy/src/{http => serverless}/conn_pool.rs (99%) rename proxy/src/{http => serverless}/sql_over_http.rs (100%) create mode 100644 proxy/src/serverless/websocket.rs rename proxy/src/{metrics.rs => usage_metrics.rs} (100%) diff --git a/proxy/src/bin/proxy.rs b/proxy/src/bin/proxy.rs index 9b54f98402..a9ca308797 100644 --- a/proxy/src/bin/proxy.rs +++ b/proxy/src/bin/proxy.rs @@ -4,10 +4,11 @@ use proxy::config::AuthenticationConfig; use proxy::config::HttpConfig; use proxy::console; use proxy::http; -use proxy::metrics; +use proxy::usage_metrics; use anyhow::bail; use proxy::config::{self, ProxyConfig}; +use proxy::serverless; use std::pin::pin; use std::{borrow::Cow, net::SocketAddr}; use tokio::net::TcpListener; @@ -129,14 +130,16 @@ async fn main() -> anyhow::Result<()> { cancellation_token.clone(), )); - if let Some(wss_address) = args.wss { - let wss_address: SocketAddr = wss_address.parse()?; - info!("Starting wss on {wss_address}"); - let wss_listener = TcpListener::bind(wss_address).await?; + // TODO: rename the argument to something like serverless. + // It now covers more than just websockets, it also covers SQL over HTTP. + if let Some(serverless_address) = args.wss { + let serverless_address: SocketAddr = serverless_address.parse()?; + info!("Starting wss on {serverless_address}"); + let serverless_listener = TcpListener::bind(serverless_address).await?; - client_tasks.spawn(http::websocket::task_main( + client_tasks.spawn(serverless::task_main( config, - wss_listener, + serverless_listener, cancellation_token.clone(), )); } @@ -144,11 +147,11 @@ async fn main() -> anyhow::Result<()> { // maintenance tasks. these never return unless there's an error let mut maintenance_tasks = JoinSet::new(); maintenance_tasks.spawn(proxy::handle_signals(cancellation_token)); - maintenance_tasks.spawn(http::server::task_main(http_listener)); + maintenance_tasks.spawn(http::health_server::task_main(http_listener)); maintenance_tasks.spawn(console::mgmt::task_main(mgmt_listener)); if let Some(metrics_config) = &config.metric_collection { - maintenance_tasks.spawn(metrics::task_main(metrics_config)); + maintenance_tasks.spawn(usage_metrics::task_main(metrics_config)); } let maintenance = loop { diff --git a/proxy/src/http.rs b/proxy/src/http.rs index 755137807b..14a9072a45 100644 --- a/proxy/src/http.rs +++ b/proxy/src/http.rs @@ -2,10 +2,7 @@ //! Other modules should use stuff from this module instead of //! directly relying on deps like `reqwest` (think loose coupling). -pub mod conn_pool; -pub mod server; -pub mod sql_over_http; -pub mod websocket; +pub mod health_server; use std::{sync::Arc, time::Duration}; diff --git a/proxy/src/http/server.rs b/proxy/src/http/health_server.rs similarity index 100% rename from proxy/src/http/server.rs rename to proxy/src/http/health_server.rs diff --git a/proxy/src/lib.rs b/proxy/src/lib.rs index a3d1cdd3c8..803b278482 100644 --- a/proxy/src/lib.rs +++ b/proxy/src/lib.rs @@ -14,14 +14,15 @@ pub mod console; pub mod error; pub mod http; pub mod logging; -pub mod metrics; pub mod parse; pub mod protocol2; pub mod proxy; pub mod sasl; pub mod scram; +pub mod serverless; pub mod stream; pub mod url; +pub mod usage_metrics; pub mod waiters; /// Handle unix signals appropriately. diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index ec878b696e..165794414f 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -8,9 +8,9 @@ use crate::{ config::{AuthenticationConfig, ProxyConfig, TlsConfig}, console::{self, errors::WakeComputeError, messages::MetricsAuxInfo, Api}, http::StatusCode, - metrics::{Ids, USAGE_METRICS}, protocol2::WithClientIp, stream::{PqStream, Stream}, + usage_metrics::{Ids, USAGE_METRICS}, }; use anyhow::{bail, Context}; use async_trait::async_trait; diff --git a/proxy/src/http/websocket.rs b/proxy/src/serverless.rs similarity index 51% rename from proxy/src/http/websocket.rs rename to proxy/src/serverless.rs index 689a84969c..23deda3ae6 100644 --- a/proxy/src/http/websocket.rs +++ b/proxy/src/serverless.rs @@ -1,235 +1,36 @@ -use crate::{ - cancellation::CancelMap, - config::ProxyConfig, - error::io_error, - protocol2::{ProxyProtocolAccept, WithClientIp}, - proxy::{ - handle_client, ClientMode, NUM_CLIENT_CONNECTION_CLOSED_COUNTER, - NUM_CLIENT_CONNECTION_OPENED_COUNTER, - }, -}; +//! Routers for our serverless APIs +//! +//! Handles both SQL over HTTP and SQL over Websockets. + +mod conn_pool; +mod sql_over_http; +mod websocket; + use anyhow::bail; -use bytes::{Buf, Bytes}; -use futures::{Sink, Stream, StreamExt}; +use hyper::StatusCode; +pub use reqwest_middleware::{ClientWithMiddleware, Error}; +pub use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; + +use crate::protocol2::{ProxyProtocolAccept, WithClientIp}; +use crate::proxy::{NUM_CLIENT_CONNECTION_CLOSED_COUNTER, NUM_CLIENT_CONNECTION_OPENED_COUNTER}; +use crate::{cancellation::CancelMap, config::ProxyConfig}; +use futures::StreamExt; use hyper::{ server::{ accept, conn::{AddrIncoming, AddrStream}, }, - upgrade::Upgraded, - Body, Method, Request, Response, StatusCode, + Body, Method, Request, Response, }; -use hyper_tungstenite::{tungstenite::Message, HyperWebsocket, WebSocketStream}; -use pin_project_lite::pin_project; -use std::{ - future::ready, - pin::Pin, - sync::Arc, - task::{ready, Context, Poll}, -}; +use std::task::Poll; +use std::{future::ready, sync::Arc}; use tls_listener::TlsListener; -use tokio::{ - io::{self, AsyncBufRead, AsyncRead, AsyncWrite, ReadBuf}, - net::TcpListener, -}; +use tokio::net::TcpListener; use tokio_util::sync::CancellationToken; use tracing::{error, info, info_span, warn, Instrument}; use utils::http::{error::ApiError, json::json_response}; -// TODO: use `std::sync::Exclusive` once it's stabilized. -// Tracking issue: https://github.com/rust-lang/rust/issues/98407. -use sync_wrapper::SyncWrapper; - -use super::{conn_pool::GlobalConnPool, sql_over_http}; - -pin_project! { - /// This is a wrapper around a [`WebSocketStream`] that - /// implements [`AsyncRead`] and [`AsyncWrite`]. - pub struct WebSocketRw { - #[pin] - stream: SyncWrapper>, - bytes: Bytes, - } -} - -impl WebSocketRw { - pub fn new(stream: WebSocketStream) -> Self { - Self { - stream: stream.into(), - bytes: Bytes::new(), - } - } -} - -impl AsyncWrite for WebSocketRw { - fn poll_write( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - let mut stream = self.project().stream.get_pin_mut(); - - ready!(stream.as_mut().poll_ready(cx).map_err(io_error))?; - match stream.as_mut().start_send(Message::Binary(buf.into())) { - Ok(()) => Poll::Ready(Ok(buf.len())), - Err(e) => Poll::Ready(Err(io_error(e))), - } - } - - fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let stream = self.project().stream.get_pin_mut(); - stream.poll_flush(cx).map_err(io_error) - } - - fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let stream = self.project().stream.get_pin_mut(); - stream.poll_close(cx).map_err(io_error) - } -} - -impl AsyncRead for WebSocketRw { - fn poll_read( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> Poll> { - if buf.remaining() > 0 { - let bytes = ready!(self.as_mut().poll_fill_buf(cx))?; - let len = std::cmp::min(bytes.len(), buf.remaining()); - buf.put_slice(&bytes[..len]); - self.consume(len); - } - - Poll::Ready(Ok(())) - } -} - -impl AsyncBufRead for WebSocketRw { - fn poll_fill_buf(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - // Please refer to poll_fill_buf's documentation. - const EOF: Poll> = Poll::Ready(Ok(&[])); - - let mut this = self.project(); - loop { - if !this.bytes.chunk().is_empty() { - let chunk = (*this.bytes).chunk(); - return Poll::Ready(Ok(chunk)); - } - - let res = ready!(this.stream.as_mut().get_pin_mut().poll_next(cx)); - match res.transpose().map_err(io_error)? { - Some(message) => match message { - Message::Ping(_) => {} - Message::Pong(_) => {} - Message::Text(text) => { - // We expect to see only binary messages. - let error = "unexpected text message in the websocket"; - warn!(length = text.len(), error); - return Poll::Ready(Err(io_error(error))); - } - Message::Frame(_) => { - // This case is impossible according to Frame's doc. - panic!("unexpected raw frame in the websocket"); - } - Message::Binary(chunk) => { - assert!(this.bytes.is_empty()); - *this.bytes = Bytes::from(chunk); - } - Message::Close(_) => return EOF, - }, - None => return EOF, - } - } - } - - fn consume(self: Pin<&mut Self>, amount: usize) { - self.project().bytes.advance(amount); - } -} - -async fn serve_websocket( - websocket: HyperWebsocket, - config: &'static ProxyConfig, - cancel_map: &CancelMap, - session_id: uuid::Uuid, - hostname: Option, -) -> anyhow::Result<()> { - let websocket = websocket.await?; - handle_client( - config, - cancel_map, - session_id, - WebSocketRw::new(websocket), - ClientMode::Websockets { hostname }, - ) - .await?; - Ok(()) -} - -async fn ws_handler( - mut request: Request, - config: &'static ProxyConfig, - conn_pool: Arc, - cancel_map: Arc, - session_id: uuid::Uuid, - sni_hostname: Option, -) -> Result, ApiError> { - let host = request - .headers() - .get("host") - .and_then(|h| h.to_str().ok()) - .and_then(|h| h.split(':').next()) - .map(|s| s.to_string()); - - // Check if the request is a websocket upgrade request. - if hyper_tungstenite::is_upgrade_request(&request) { - info!(session_id = ?session_id, "performing websocket upgrade"); - - let (response, websocket) = hyper_tungstenite::upgrade(&mut request, None) - .map_err(|e| ApiError::BadRequest(e.into()))?; - - tokio::spawn( - async move { - if let Err(e) = - serve_websocket(websocket, config, &cancel_map, session_id, host).await - { - error!(session_id = ?session_id, "error in websocket connection: {e:#}"); - } - } - .in_current_span(), - ); - - // Return the response so the spawned future can continue. - Ok(response) - // TODO: that deserves a refactor as now this function also handles http json client besides websockets. - // Right now I don't want to blow up sql-over-http patch with file renames and do that as a follow up instead. - } else if request.uri().path() == "/sql" && request.method() == Method::POST { - sql_over_http::handle( - request, - sni_hostname, - conn_pool, - session_id, - &config.http_config, - ) - .await - } else if request.uri().path() == "/sql" && request.method() == Method::OPTIONS { - Response::builder() - .header("Allow", "OPTIONS, POST") - .header("Access-Control-Allow-Origin", "*") - .header( - "Access-Control-Allow-Headers", - "Neon-Connection-String, Neon-Raw-Text-Output, Neon-Array-Mode, Neon-Pool-Opt-In", - ) - .header("Access-Control-Max-Age", "86400" /* 24 hours */) - .status(StatusCode::OK) // 204 is also valid, but see: https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/OPTIONS#status_code - .body(Body::empty()) - .map_err(|e| ApiError::InternalServerError(e.into())) - } else { - json_response(StatusCode::BAD_REQUEST, "query is not supported") - } -} - pub async fn task_main( config: &'static ProxyConfig, ws_listener: TcpListener, @@ -239,7 +40,7 @@ pub async fn task_main( info!("websocket server has shut down"); } - let conn_pool: Arc = GlobalConnPool::new(config); + let conn_pool = conn_pool::GlobalConnPool::new(config); // shutdown the connection pool tokio::spawn({ @@ -300,13 +101,15 @@ pub async fn task_main( let cancel_map = Arc::new(CancelMap::default()); let session_id = uuid::Uuid::new_v4(); - ws_handler(req, config, conn_pool, cancel_map, session_id, sni_name) - .instrument(info_span!( - "ws-client", - session = %session_id, - %peer_addr, - )) - .await + request_handler( + req, config, conn_pool, cancel_map, session_id, sni_name, + ) + .instrument(info_span!( + "serverless", + session = %session_id, + %peer_addr, + )) + .await } }, ))) @@ -359,3 +162,65 @@ where self.inner.call(req) } } + +async fn request_handler( + mut request: Request, + config: &'static ProxyConfig, + conn_pool: Arc, + cancel_map: Arc, + session_id: uuid::Uuid, + sni_hostname: Option, +) -> Result, ApiError> { + let host = request + .headers() + .get("host") + .and_then(|h| h.to_str().ok()) + .and_then(|h| h.split(':').next()) + .map(|s| s.to_string()); + + // Check if the request is a websocket upgrade request. + if hyper_tungstenite::is_upgrade_request(&request) { + info!(session_id = ?session_id, "performing websocket upgrade"); + + let (response, websocket) = hyper_tungstenite::upgrade(&mut request, None) + .map_err(|e| ApiError::BadRequest(e.into()))?; + + tokio::spawn( + async move { + if let Err(e) = + websocket::serve_websocket(websocket, config, &cancel_map, session_id, host) + .await + { + error!(session_id = ?session_id, "error in websocket connection: {e:#}"); + } + } + .in_current_span(), + ); + + // Return the response so the spawned future can continue. + Ok(response) + } else if request.uri().path() == "/sql" && request.method() == Method::POST { + sql_over_http::handle( + request, + sni_hostname, + conn_pool, + session_id, + &config.http_config, + ) + .await + } else if request.uri().path() == "/sql" && request.method() == Method::OPTIONS { + Response::builder() + .header("Allow", "OPTIONS, POST") + .header("Access-Control-Allow-Origin", "*") + .header( + "Access-Control-Allow-Headers", + "Neon-Connection-String, Neon-Raw-Text-Output, Neon-Array-Mode, Neon-Pool-Opt-In", + ) + .header("Access-Control-Max-Age", "86400" /* 24 hours */) + .status(StatusCode::OK) // 204 is also valid, but see: https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/OPTIONS#status_code + .body(Body::empty()) + .map_err(|e| ApiError::InternalServerError(e.into())) + } else { + json_response(StatusCode::BAD_REQUEST, "query is not supported") + } +} diff --git a/proxy/src/http/conn_pool.rs b/proxy/src/serverless/conn_pool.rs similarity index 99% rename from proxy/src/http/conn_pool.rs rename to proxy/src/serverless/conn_pool.rs index 5218a44479..c5bfc32568 100644 --- a/proxy/src/http/conn_pool.rs +++ b/proxy/src/serverless/conn_pool.rs @@ -22,8 +22,8 @@ use tokio_postgres::{AsyncMessage, ReadyForQueryStatus}; use crate::{ auth, console, - metrics::{Ids, MetricCounter, USAGE_METRICS}, proxy::{LatencyTimer, NUM_DB_CONNECTIONS_CLOSED_COUNTER, NUM_DB_CONNECTIONS_OPENED_COUNTER}, + usage_metrics::{Ids, MetricCounter, USAGE_METRICS}, }; use crate::{compute, config}; diff --git a/proxy/src/http/sql_over_http.rs b/proxy/src/serverless/sql_over_http.rs similarity index 100% rename from proxy/src/http/sql_over_http.rs rename to proxy/src/serverless/sql_over_http.rs diff --git a/proxy/src/serverless/websocket.rs b/proxy/src/serverless/websocket.rs new file mode 100644 index 0000000000..86141ab64f --- /dev/null +++ b/proxy/src/serverless/websocket.rs @@ -0,0 +1,146 @@ +use crate::{ + cancellation::CancelMap, + config::ProxyConfig, + error::io_error, + proxy::{handle_client, ClientMode}, +}; +use bytes::{Buf, Bytes}; +use futures::{Sink, Stream}; +use hyper::upgrade::Upgraded; +use hyper_tungstenite::{tungstenite::Message, HyperWebsocket, WebSocketStream}; +use pin_project_lite::pin_project; + +use std::{ + pin::Pin, + task::{ready, Context, Poll}, +}; +use tokio::io::{self, AsyncBufRead, AsyncRead, AsyncWrite, ReadBuf}; +use tracing::warn; + +// TODO: use `std::sync::Exclusive` once it's stabilized. +// Tracking issue: https://github.com/rust-lang/rust/issues/98407. +use sync_wrapper::SyncWrapper; + +pin_project! { + /// This is a wrapper around a [`WebSocketStream`] that + /// implements [`AsyncRead`] and [`AsyncWrite`]. + pub struct WebSocketRw { + #[pin] + stream: SyncWrapper>, + bytes: Bytes, + } +} + +impl WebSocketRw { + pub fn new(stream: WebSocketStream) -> Self { + Self { + stream: stream.into(), + bytes: Bytes::new(), + } + } +} + +impl AsyncWrite for WebSocketRw { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + let mut stream = self.project().stream.get_pin_mut(); + + ready!(stream.as_mut().poll_ready(cx).map_err(io_error))?; + match stream.as_mut().start_send(Message::Binary(buf.into())) { + Ok(()) => Poll::Ready(Ok(buf.len())), + Err(e) => Poll::Ready(Err(io_error(e))), + } + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let stream = self.project().stream.get_pin_mut(); + stream.poll_flush(cx).map_err(io_error) + } + + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let stream = self.project().stream.get_pin_mut(); + stream.poll_close(cx).map_err(io_error) + } +} + +impl AsyncRead for WebSocketRw { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + if buf.remaining() > 0 { + let bytes = ready!(self.as_mut().poll_fill_buf(cx))?; + let len = std::cmp::min(bytes.len(), buf.remaining()); + buf.put_slice(&bytes[..len]); + self.consume(len); + } + + Poll::Ready(Ok(())) + } +} + +impl AsyncBufRead for WebSocketRw { + fn poll_fill_buf(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + // Please refer to poll_fill_buf's documentation. + const EOF: Poll> = Poll::Ready(Ok(&[])); + + let mut this = self.project(); + loop { + if !this.bytes.chunk().is_empty() { + let chunk = (*this.bytes).chunk(); + return Poll::Ready(Ok(chunk)); + } + + let res = ready!(this.stream.as_mut().get_pin_mut().poll_next(cx)); + match res.transpose().map_err(io_error)? { + Some(message) => match message { + Message::Ping(_) => {} + Message::Pong(_) => {} + Message::Text(text) => { + // We expect to see only binary messages. + let error = "unexpected text message in the websocket"; + warn!(length = text.len(), error); + return Poll::Ready(Err(io_error(error))); + } + Message::Frame(_) => { + // This case is impossible according to Frame's doc. + panic!("unexpected raw frame in the websocket"); + } + Message::Binary(chunk) => { + assert!(this.bytes.is_empty()); + *this.bytes = Bytes::from(chunk); + } + Message::Close(_) => return EOF, + }, + None => return EOF, + } + } + } + + fn consume(self: Pin<&mut Self>, amount: usize) { + self.project().bytes.advance(amount); + } +} + +pub async fn serve_websocket( + websocket: HyperWebsocket, + config: &'static ProxyConfig, + cancel_map: &CancelMap, + session_id: uuid::Uuid, + hostname: Option, +) -> anyhow::Result<()> { + let websocket = websocket.await?; + handle_client( + config, + cancel_map, + session_id, + WebSocketRw::new(websocket), + ClientMode::Websockets { hostname }, + ) + .await?; + Ok(()) +} diff --git a/proxy/src/metrics.rs b/proxy/src/usage_metrics.rs similarity index 100% rename from proxy/src/metrics.rs rename to proxy/src/usage_metrics.rs From c155cc0c3f7143a12ede9ce05f1873fbcddec645 Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Wed, 25 Oct 2023 11:53:04 -0400 Subject: [PATCH 15/69] Fix test instructions readme (#5644) --- test_runner/performance/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test_runner/performance/README.md b/test_runner/performance/README.md index c1a57fb28b..d113e9e637 100644 --- a/test_runner/performance/README.md +++ b/test_runner/performance/README.md @@ -4,8 +4,10 @@ First make a release build. The `-s` flag silences a lot of output, and makes it easier to see if you have compile errors without scrolling up. `BUILD_TYPE=release CARGO_BUILD_FLAGS="--features=testing" make -s -j8` +You may also need to run `./scripts/pysync`. + Then run the tests -`NEON_BIN=./target/release poetry run pytest test_runner/performance"` +`DEFAULT_PG_VERSION=15 NEON_BIN=./target/release poetry run pytest test_runner/performance` Some handy pytest flags for local development: - `-x` tells pytest to stop on first error From a673e4e7a915e0a26b392ac24f80f6318a60413f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Wed, 25 Oct 2023 18:46:34 +0200 Subject: [PATCH 16/69] Optionally return json from get_lsn_by_timestamp (#5608) This does two things: first a minor refactor to not use HTTP/1.x style header names and also to not panic if some certain requests had no "Accept" header. As a second thing, it addresses the third bullet point from #3689: > Change `get_lsn_by_timestamp` API method to return LSN even if we only found commit before the specified timestamp. This is done by adding a version parameter to the `get_lsn_by_timestamp` API call and making its behaviour depend on the version number. Part of #3414 (but doesn't address it in its entirety). --------- Co-authored-by: Joonas Koivunen --- pageserver/src/http/openapi_spec.yml | 21 ++++++- pageserver/src/http/routes.rs | 50 ++++++++++++---- test_runner/fixtures/pageserver/http.py | 4 +- test_runner/regress/test_lsn_mapping.py | 78 +++++++++++++++++++++++-- 4 files changed, 132 insertions(+), 21 deletions(-) diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index f0bf2666a7..e8421d619d 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -392,13 +392,19 @@ paths: type: string format: date-time description: A timestamp to get the LSN + - name: version + in: query + required: false + schema: + type: integer + description: The version of the endpoint to use responses: "200": description: OK content: application/json: schema: - type: string + $ref: "#/components/schemas/LsnByTimestampResponse" "400": description: Error when no tenant id found in path, no timeline id or invalid timestamp content: @@ -1384,6 +1390,19 @@ components: type: string format: hex + LsnByTimestampResponse: + type: object + required: + - lsn + - kind + properties: + lsn: + type: string + format: hex + kind: + type: string + enum: [past, present, future, nodata] + Error: type: object required: diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 142e85d203..54c5cdf133 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use anyhow::{anyhow, Context, Result}; use futures::TryFutureExt; use humantime::format_rfc3339; -use hyper::header::CONTENT_TYPE; +use hyper::header; use hyper::StatusCode; use hyper::{Body, Request, Response, Uri}; use metrics::launch_timestamp::LaunchTimestamp; @@ -17,6 +17,7 @@ use pageserver_api::models::{ TenantLoadRequest, TenantLocationConfigRequest, }; use remote_storage::GenericRemoteStorage; +use serde_with::{serde_as, DisplayFromStr}; use tenant_size_model::{SizeResult, StorageModel}; use tokio_util::sync::CancellationToken; use tracing::*; @@ -484,6 +485,8 @@ async fn get_lsn_by_timestamp_handler( let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; + let version: Option = parse_query_param(&request, "version")?; + let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; let timestamp_raw = must_get_query_param(&request, "timestamp")?; let timestamp = humantime::parse_rfc3339(×tamp_raw) @@ -495,13 +498,32 @@ async fn get_lsn_by_timestamp_handler( let timeline = active_timeline_of_active_tenant(tenant_id, timeline_id).await?; let result = timeline.find_lsn_for_timestamp(timestamp_pg, &ctx).await?; - let result = match result { - LsnForTimestamp::Present(lsn) => format!("{lsn}"), - LsnForTimestamp::Future(_lsn) => "future".into(), - LsnForTimestamp::Past(_lsn) => "past".into(), - LsnForTimestamp::NoData(_lsn) => "nodata".into(), - }; - json_response(StatusCode::OK, result) + if version.unwrap_or(0) > 1 { + #[serde_as] + #[derive(serde::Serialize)] + struct Result { + #[serde_as(as = "DisplayFromStr")] + lsn: Lsn, + kind: &'static str, + } + let (lsn, kind) = match result { + LsnForTimestamp::Present(lsn) => (lsn, "present"), + LsnForTimestamp::Future(lsn) => (lsn, "future"), + LsnForTimestamp::Past(lsn) => (lsn, "past"), + LsnForTimestamp::NoData(lsn) => (lsn, "nodata"), + }; + json_response(StatusCode::OK, Result { lsn, kind }) + } else { + // FIXME: this is a temporary crutch not to break backwards compatibility + // See https://github.com/neondatabase/neon/pull/5608 + let result = match result { + LsnForTimestamp::Present(lsn) => format!("{lsn}"), + LsnForTimestamp::Future(_lsn) => "future".into(), + LsnForTimestamp::Past(_lsn) => "past".into(), + LsnForTimestamp::NoData(_lsn) => "nodata".into(), + }; + json_response(StatusCode::OK, result) + } } async fn get_timestamp_of_lsn_handler( @@ -767,6 +789,10 @@ async fn tenant_size_handler( .map_err(ApiError::InternalServerError)?; let mut sizes = None; + let accepts_html = headers + .get(header::ACCEPT) + .map(|v| v == "text/html") + .unwrap_or_default(); if !inputs_only.unwrap_or(false) { let storage_model = inputs .calculate_model() @@ -774,11 +800,11 @@ async fn tenant_size_handler( let size = storage_model.calculate(); // If request header expects html, return html - if headers["Accept"] == "text/html" { + if accepts_html { return synthetic_size_html_response(inputs, storage_model, size); } sizes = Some(size); - } else if headers["Accept"] == "text/html" { + } else if accepts_html { return Err(ApiError::BadRequest(anyhow!( "inputs_only parameter is incompatible with html output request" ))); @@ -929,7 +955,7 @@ fn synthetic_size_html_response( pub fn html_response(status: StatusCode, data: String) -> Result, ApiError> { let response = Response::builder() .status(status) - .header(hyper::header::CONTENT_TYPE, "text/html") + .header(header::CONTENT_TYPE, "text/html") .body(Body::from(data.as_bytes().to_vec())) .map_err(|e| ApiError::InternalServerError(e.into()))?; Ok(response) @@ -1310,7 +1336,7 @@ async fn getpage_at_lsn_handler( Result::<_, ApiError>::Ok( Response::builder() .status(StatusCode::OK) - .header(CONTENT_TYPE, "application/octet-stream") + .header(header::CONTENT_TYPE, "application/octet-stream") .body(hyper::Body::from(page)) .unwrap(), ) diff --git a/test_runner/fixtures/pageserver/http.py b/test_runner/fixtures/pageserver/http.py index 598b48b56f..d2c3715c52 100644 --- a/test_runner/fixtures/pageserver/http.py +++ b/test_runner/fixtures/pageserver/http.py @@ -441,13 +441,13 @@ class PageserverHttpClient(requests.Session): assert res_json is None def timeline_get_lsn_by_timestamp( - self, tenant_id: TenantId, timeline_id: TimelineId, timestamp + self, tenant_id: TenantId, timeline_id: TimelineId, timestamp, version: int ): log.info( f"Requesting lsn by timestamp {timestamp}, tenant {tenant_id}, timeline {timeline_id}" ) res = self.get( - f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/get_lsn_by_timestamp?timestamp={timestamp}", + f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/get_lsn_by_timestamp?timestamp={timestamp}&version={version}", ) self.verbose_error(res) res_json = res.json() diff --git a/test_runner/regress/test_lsn_mapping.py b/test_runner/regress/test_lsn_mapping.py index 03606e3c1c..726bfa5f29 100644 --- a/test_runner/regress/test_lsn_mapping.py +++ b/test_runner/regress/test_lsn_mapping.py @@ -8,6 +8,71 @@ from fixtures.types import Lsn from fixtures.utils import query_scalar +# +# Test pageserver get_lsn_by_timestamp API +# +def test_lsn_mapping_old(neon_env_builder: NeonEnvBuilder): + env = neon_env_builder.init_start() + + new_timeline_id = env.neon_cli.create_branch("test_lsn_mapping") + endpoint_main = env.endpoints.create_start("test_lsn_mapping") + log.info("postgres is running on 'test_lsn_mapping' branch") + + cur = endpoint_main.connect().cursor() + # Create table, and insert rows, each in a separate transaction + # Disable synchronous_commit to make this initialization go faster. + # + # Each row contains current insert LSN and the current timestamp, when + # the row was inserted. + cur.execute("SET synchronous_commit=off") + cur.execute("CREATE TABLE foo (x integer)") + tbl = [] + for i in range(1000): + cur.execute("INSERT INTO foo VALUES(%s)", (i,)) + # Get the timestamp at UTC + after_timestamp = query_scalar(cur, "SELECT clock_timestamp()").replace(tzinfo=None) + tbl.append([i, after_timestamp]) + + # Execute one more transaction with synchronous_commit enabled, to flush + # all the previous transactions + cur.execute("SET synchronous_commit=on") + cur.execute("INSERT INTO foo VALUES (-1)") + + # Wait until WAL is received by pageserver + wait_for_last_flush_lsn(env, endpoint_main, env.initial_tenant, new_timeline_id) + + with env.pageserver.http_client() as client: + # Check edge cases: timestamp in the future + probe_timestamp = tbl[-1][1] + timedelta(hours=1) + result = client.timeline_get_lsn_by_timestamp( + env.initial_tenant, new_timeline_id, f"{probe_timestamp.isoformat()}Z", 1 + ) + assert result == "future" + + # timestamp too the far history + probe_timestamp = tbl[0][1] - timedelta(hours=10) + result = client.timeline_get_lsn_by_timestamp( + env.initial_tenant, new_timeline_id, f"{probe_timestamp.isoformat()}Z", 1 + ) + assert result == "past" + + # Probe a bunch of timestamps in the valid range + for i in range(1, len(tbl), 100): + probe_timestamp = tbl[i][1] + lsn = client.timeline_get_lsn_by_timestamp( + env.initial_tenant, new_timeline_id, f"{probe_timestamp.isoformat()}Z", 1 + ) + # Call get_lsn_by_timestamp to get the LSN + # Launch a new read-only node at that LSN, and check that only the rows + # that were supposed to be committed at that point in time are visible. + endpoint_here = env.endpoints.create_start( + branch_name="test_lsn_mapping", endpoint_id="ep-lsn_mapping_read", lsn=lsn + ) + assert endpoint_here.safe_psql("SELECT max(x) FROM foo")[0][0] == i + + endpoint_here.stop_and_destroy() + + # # Test pageserver get_lsn_by_timestamp API # @@ -45,23 +110,24 @@ def test_lsn_mapping(neon_env_builder: NeonEnvBuilder): # Check edge cases: timestamp in the future probe_timestamp = tbl[-1][1] + timedelta(hours=1) result = client.timeline_get_lsn_by_timestamp( - env.initial_tenant, new_timeline_id, f"{probe_timestamp.isoformat()}Z" + env.initial_tenant, new_timeline_id, f"{probe_timestamp.isoformat()}Z", 2 ) - assert result == "future" + assert result["kind"] == "future" # timestamp too the far history probe_timestamp = tbl[0][1] - timedelta(hours=10) result = client.timeline_get_lsn_by_timestamp( - env.initial_tenant, new_timeline_id, f"{probe_timestamp.isoformat()}Z" + env.initial_tenant, new_timeline_id, f"{probe_timestamp.isoformat()}Z", 2 ) - assert result == "past" + assert result["kind"] == "past" # Probe a bunch of timestamps in the valid range for i in range(1, len(tbl), 100): probe_timestamp = tbl[i][1] - lsn = client.timeline_get_lsn_by_timestamp( - env.initial_tenant, new_timeline_id, f"{probe_timestamp.isoformat()}Z" + result = client.timeline_get_lsn_by_timestamp( + env.initial_tenant, new_timeline_id, f"{probe_timestamp.isoformat()}Z", 2 ) + lsn = result["lsn"] # Call get_lsn_by_timestamp to get the LSN # Launch a new read-only node at that LSN, and check that only the rows # that were supposed to be committed at that point in time are visible. From 4ddbc0e46d02c43faeadd7de20130d27e17f7e78 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 25 Oct 2023 20:35:23 +0300 Subject: [PATCH 17/69] Ignore missed AUX_FILES_KEY when generating image layer (#5660) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem Logical replication requires new AUX_FILES_KEY which is definitely absent in existed database. We do not have function to check if key exists in our KV storage. So I have to handle the error in `list_aux_files` method. But this key is also included in key space range and accessed y `create_image_layer` method. ## Summary of changes Check if AUX_FILES_KEY exists before including it in keyspace. --------- Co-authored-by: Konstantin Knizhnik Co-authored-by: Shany Pozin Co-authored-by: Arpad Müller --- pageserver/src/pgdatadir_mapping.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index d27c8a3d5d..52c44746dc 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -675,8 +675,9 @@ impl Timeline { result.add_key(CONTROLFILE_KEY); result.add_key(CHECKPOINT_KEY); - result.add_key(AUX_FILES_KEY); - + if self.get(AUX_FILES_KEY, lsn, ctx).await.is_ok() { + result.add_key(AUX_FILES_KEY); + } Ok(result.to_keyspace()) } From 325258413ae1711a985d3bac299927fe5cdb906a Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Wed, 25 Oct 2023 21:29:12 +0300 Subject: [PATCH 18/69] fix: trampling on global physical size metric (#5663) All loading (attached, or from disk) timelines overwrite the global gauge for physical size. The `_set` method cannot be used safely, so remove it and just "add" the physical size. --- pageserver/src/metrics.rs | 11 +++-------- pageserver/src/tenant/timeline.rs | 2 +- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index eea3de0583..267e632312 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1388,27 +1388,22 @@ impl TimelineMetrics { } } - pub fn record_new_file_metrics(&self, sz: u64) { + pub(crate) fn record_new_file_metrics(&self, sz: u64) { self.resident_physical_size_add(sz); self.num_persistent_files_created.inc_by(1); self.persistent_bytes_written.inc_by(sz); } - pub fn resident_physical_size_sub(&self, sz: u64) { + pub(crate) fn resident_physical_size_sub(&self, sz: u64) { self.resident_physical_size_gauge.sub(sz); crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.sub(sz); } - pub fn resident_physical_size_add(&self, sz: u64) { + pub(crate) fn resident_physical_size_add(&self, sz: u64) { self.resident_physical_size_gauge.add(sz); crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.add(sz); } - pub fn resident_physical_size_set(&self, sz: u64) { - self.resident_physical_size_gauge.set(sz); - crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.set(sz); - } - pub fn resident_physical_size_get(&self) -> u64 { self.resident_physical_size_gauge.get() } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 201541244a..569426bbb3 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -1871,7 +1871,7 @@ impl Timeline { "loaded layer map with {} layers at {}, total physical size: {}", num_layers, disk_consistent_lsn, total_physical_size ); - self.metrics.resident_physical_size_set(total_physical_size); + self.metrics.resident_physical_size_add(total_physical_size); timer.stop_and_record(); Ok(()) From f70019797cbfd1c5205cf4b533dc25f53c0c168c Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 26 Oct 2023 00:25:43 +0300 Subject: [PATCH 19/69] refactor(rtc): schedule compaction update (#5649) a single operation instead of N uploads and 1 deletion scheduling with write(layer_map) lock releasing in the between. Compaction update will make for a much better place to change how the operation will change in future compared to more general file based operations. builds upon #5645. solves the problem of difficult to see hopeful correctness w.r.t. other `index_part.json` changing operations. Co-authored-by: Shany Pozin --- .../src/tenant/remote_timeline_client.rs | 69 +++++++++++++------ pageserver/src/tenant/timeline.rs | 24 +++---- 2 files changed, 60 insertions(+), 33 deletions(-) diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index cad86fb4aa..e8415ee9f3 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -627,7 +627,7 @@ impl RemoteTimelineClient { /// /// Launch an upload operation in the background. /// - pub fn schedule_layer_file_upload( + pub(crate) fn schedule_layer_file_upload( self: &Arc, layer_file_name: &LayerFileName, layer_metadata: &LayerFileMetadata, @@ -635,6 +635,17 @@ impl RemoteTimelineClient { let mut guard = self.upload_queue.lock().unwrap(); let upload_queue = guard.initialized_mut()?; + self.schedule_layer_file_upload0(upload_queue, layer_file_name, layer_metadata); + self.launch_queued_tasks(upload_queue); + Ok(()) + } + + fn schedule_layer_file_upload0( + self: &Arc, + upload_queue: &mut UploadQueueInitialized, + layer_file_name: &LayerFileName, + layer_metadata: &LayerFileMetadata, + ) { upload_queue .latest_files .insert(layer_file_name.clone(), layer_metadata.clone()); @@ -643,12 +654,7 @@ impl RemoteTimelineClient { let op = UploadOp::UploadLayer(layer_file_name.clone(), layer_metadata.clone()); self.calls_unfinished_metric_begin(&op); upload_queue.queued_operations.push_back(op); - info!("scheduled layer file upload {layer_file_name}"); - - // Launch the task immediately, if possible - self.launch_queued_tasks(upload_queue); - Ok(()) } /// Launch a delete operation in the background. @@ -666,16 +672,13 @@ impl RemoteTimelineClient { let mut guard = self.upload_queue.lock().unwrap(); let upload_queue = guard.initialized_mut()?; - let no_bail_here = || { - let with_generations = - self.schedule_unlinking_of_layers_from_index_part0(upload_queue, names); + let with_generations = + self.schedule_unlinking_of_layers_from_index_part0(upload_queue, &names); - self.schedule_deletion_of_unlinked0(upload_queue, with_generations); + self.schedule_deletion_of_unlinked0(upload_queue, with_generations); - // Launch the tasks immediately, if possible - self.launch_queued_tasks(upload_queue); - }; - no_bail_here(); + // Launch the tasks immediately, if possible + self.launch_queued_tasks(upload_queue); Ok(()) } @@ -694,7 +697,7 @@ impl RemoteTimelineClient { // just forget the return value; after uploading the next index_part.json, we can consider // the layer files as "dangling". this is fine however. - self.schedule_unlinking_of_layers_from_index_part0(upload_queue, names); + self.schedule_unlinking_of_layers_from_index_part0(upload_queue, &names); self.launch_queued_tasks(upload_queue); @@ -706,7 +709,7 @@ impl RemoteTimelineClient { fn schedule_unlinking_of_layers_from_index_part0( self: &Arc, upload_queue: &mut UploadQueueInitialized, - names: Vec, + names: &[LayerFileName], ) -> Vec<(LayerFileName, Generation)> { // Deleting layers doesn't affect the values stored in TimelineMetadata, // so we don't need update it. Just serialize it. @@ -715,14 +718,14 @@ impl RemoteTimelineClient { // Decorate our list of names with each name's generation, dropping // makes that are unexpectedly missing from our metadata. let with_generations: Vec<_> = names - .into_iter() + .iter() .filter_map(|name| { // Remove from latest_files, learning the file's remote generation in the process - let meta = upload_queue.latest_files.remove(&name); + let meta = upload_queue.latest_files.remove(name); if let Some(meta) = meta { upload_queue.latest_files_changes_since_metadata_upload_scheduled += 1; - Some((name, meta.generation)) + Some((name.to_owned(), meta.generation)) } else { // This can only happen if we forgot to to schedule the file upload // before scheduling the delete. Log it because it is a rare/strange @@ -734,6 +737,9 @@ impl RemoteTimelineClient { }) .collect(); + // after unlinking files from the upload_queue.latest_files we must always schedule an + // index_part update, because that needs to be uploaded before we can actually delete the + // files. if upload_queue.latest_files_changes_since_metadata_upload_scheduled > 0 { self.schedule_index_upload(upload_queue, metadata); } @@ -742,7 +748,7 @@ impl RemoteTimelineClient { } /// Schedules deletion for layer files which have previously been unlinked from the - /// `index_part.json`. + /// `index_part.json` with [`Self::schedule_unlinking_of_layers_from_index_part`]. #[allow(unused)] // will be used by Layer::drop in PR#4938 pub(crate) fn schedule_deletion_of_unlinked( self: &Arc, @@ -773,6 +779,29 @@ impl RemoteTimelineClient { upload_queue.queued_operations.push_back(op); } + /// Schedules a compaction update to the remote `index_part.json`. + /// + /// `compacted_from` represent the L0 names which have been `compacted_to` L1 layers. + pub(crate) fn schedule_compaction_update( + self: &Arc, + compacted_from: &[LayerFileName], + compacted_to: &[(LayerFileName, LayerFileMetadata)], + ) -> anyhow::Result<()> { + let mut guard = self.upload_queue.lock().unwrap(); + let upload_queue = guard.initialized_mut()?; + + for (name, m) in compacted_to { + self.schedule_layer_file_upload0(upload_queue, name, m); + } + + let with_generations = + self.schedule_unlinking_of_layers_from_index_part0(upload_queue, compacted_from); + self.schedule_deletion_of_unlinked0(upload_queue, with_generations); + self.launch_queued_tasks(upload_queue); + + Ok(()) + } + /// /// Wait for all previously scheduled uploads/deletions to complete /// diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 569426bbb3..91aad10210 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -3870,22 +3870,21 @@ impl Timeline { // now, we just skip the file to avoid unintentional modification to files on the disk and in the layer map. let mut duplicated_layers = HashSet::new(); + let mut uploaded_layers = Vec::with_capacity(new_layers.len()); let mut insert_layers = Vec::new(); let mut remove_layers = Vec::new(); - for l in new_layers { + for l in &new_layers { let new_delta_path = l.path(); let metadata = new_delta_path.metadata().with_context(|| { format!("read file metadata for new created layer {new_delta_path}") })?; - if let Some(remote_client) = &self.remote_client { - remote_client.schedule_layer_file_upload( - &l.filename(), - &LayerFileMetadata::new(metadata.len(), self.generation), - )?; - } + uploaded_layers.push(( + l.filename(), + LayerFileMetadata::new(metadata.len(), self.generation), + )); // update metrics, including the timeline's physical size self.metrics.record_new_file_metrics(metadata.len()); @@ -3898,7 +3897,7 @@ impl Timeline { LayerResidenceStatus::Resident, LayerResidenceEventReason::LayerCreate, ); - let l = l as Arc; + let l = l.to_owned() as Arc; if guard.contains(&l) { tracing::error!(layer=%l, "duplicated L1 layer"); duplicated_layers.insert(l.layer_desc().key()); @@ -3930,13 +3929,12 @@ impl Timeline { &self.metrics, )?; - drop_wlock(guard); - - // Also schedule the deletions in remote storage - if let Some(remote_client) = &self.remote_client { - remote_client.schedule_layer_file_deletion(layer_names_to_delete)?; + if let Some(remote_client) = self.remote_client.as_ref() { + remote_client.schedule_compaction_update(&layer_names_to_delete, &uploaded_layers)?; } + drop_wlock(guard); + Ok(()) } From 85f4514e7d1fb151fe6067a47563657c9df9c76c Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Wed, 25 Oct 2023 22:54:23 +0100 Subject: [PATCH 20/69] Get env var for real Azure tests from GitHub (#5662) ## Problem We'll need to switch `REMOTE_STORAGE_AZURE_REGION` from the current `eastus2` region to something `eu-central-1`-like. This may require changing `AZURE_STORAGE_ACCESS_KEY`. To make it possible to switch from one place (not to break a lot of builds on CI), move `REMOTE_STORAGE_AZURE_CONTAINER` and `REMOTE_STORAGE_AZURE_REGION` to GitHub Variables. See https://github.com/neondatabase/neon/settings/variables/actions ## Summary of changes - Get values for `REMOTE_STORAGE_AZURE_CONTAINER` & `REMOTE_STORAGE_AZURE_REGION` from GitHub Variables --- .github/actionlint.yml | 2 ++ .github/workflows/build_and_test.yml | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/actionlint.yml b/.github/actionlint.yml index fddd2f980d..e2ece5f230 100644 --- a/.github/actionlint.yml +++ b/.github/actionlint.yml @@ -5,4 +5,6 @@ self-hosted-runner: - small - us-east-2 config-variables: + - REMOTE_STORAGE_AZURE_CONTAINER + - REMOTE_STORAGE_AZURE_REGION - SLACK_UPCOMING_RELEASE_CHANNEL_ID diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 58ddc66d5d..57b5776b76 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -340,11 +340,11 @@ jobs: # Run separate tests for real Azure Blob Storage # XXX: replace region with `eu-central-1`-like region + export ENABLE_REAL_AZURE_REMOTE_STORAGE=y export AZURE_STORAGE_ACCOUNT="${{ secrets.AZURE_STORAGE_ACCOUNT_DEV }}" export AZURE_STORAGE_ACCESS_KEY="${{ secrets.AZURE_STORAGE_ACCESS_KEY_DEV }}" - export ENABLE_REAL_AZURE_REMOTE_STORAGE=y - export REMOTE_STORAGE_AZURE_CONTAINER=neon-github-sandbox - export REMOTE_STORAGE_AZURE_REGION=eastus2 + export REMOTE_STORAGE_AZURE_CONTAINER="${{ vars.REMOTE_STORAGE_AZURE_CONTAINER }}" + export REMOTE_STORAGE_AZURE_REGION="${{ vars.REMOTE_STORAGE_AZURE_REGION }}" # Avoid `$CARGO_FEATURES` since there's no `testing` feature in the e2e tests now ${cov_prefix} cargo test $CARGO_FLAGS --package remote_storage --test test_real_azure From 378daa358b63106d3972ee531ea8c197b7ece89b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Oct 2023 22:50:35 +0000 Subject: [PATCH 21/69] build(deps): bump werkzeug from 2.2.3 to 3.0.1 (#5665) --- poetry.lock | 12 ++++++------ pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 421e22b523..0beae18cdf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2447,20 +2447,20 @@ test = ["websockets"] [[package]] name = "werkzeug" -version = "2.2.3" +version = "3.0.1" description = "The comprehensive WSGI web application library." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "Werkzeug-2.2.3-py3-none-any.whl", hash = "sha256:56433961bc1f12533306c624f3be5e744389ac61d722175d543e1751285da612"}, - {file = "Werkzeug-2.2.3.tar.gz", hash = "sha256:2e1ccc9417d4da358b9de6f174e3ac094391ea1d4fbef2d667865d819dfd0afe"}, + {file = "werkzeug-3.0.1-py3-none-any.whl", hash = "sha256:90a285dc0e42ad56b34e696398b8122ee4c681833fb35b8334a095d82c56da10"}, + {file = "werkzeug-3.0.1.tar.gz", hash = "sha256:507e811ecea72b18a404947aded4b3390e1db8f826b494d76550ef45bb3b1dcc"}, ] [package.dependencies] MarkupSafe = ">=2.1.1" [package.extras] -watchdog = ["watchdog"] +watchdog = ["watchdog (>=2.3)"] [[package]] name = "wrapt" @@ -2719,4 +2719,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "c5981d8d7c2deadd47c823bc35f86f830c8e320b653d2d3718bade1f4d2dabca" +content-hash = "74649cf47c52f21b01b096a42044750b1c9677576b405be0489c2909127a9bf1" diff --git a/pyproject.toml b/pyproject.toml index 946a64288a..ebf69c636e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ backoff = "^2.2.1" pytest-lazy-fixture = "^0.6.3" prometheus-client = "^0.14.1" pytest-timeout = "^2.1.0" -Werkzeug = "^2.2.3" +Werkzeug = "^3.0.1" pytest-order = "^1.1.0" allure-pytest = "^2.13.2" pytest-asyncio = "^0.21.0" From acda65d7d4c4cdcf270efb3ff4b93e6983b9efff Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 26 Oct 2023 09:53:18 +0100 Subject: [PATCH 22/69] pageserver: quieten "Failed to get info about AUX files" (#5669) ## Problem This line caused lots of errors to be emitted for healthy tenants. ## Summary of changes Downgrade to debug, since it is an expected code path we'll take for tenants at startup. --- pageserver/src/pgdatadir_mapping.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index 52c44746dc..b611583b60 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -1202,7 +1202,8 @@ impl<'a> DatadirModification<'a> { let mut dir = match self.get(AUX_FILES_KEY, ctx).await { Ok(buf) => AuxFilesDirectory::des(&buf)?, Err(e) => { - warn!("Failed to get info about AUX files: {}", e); + // This is expected: historical databases do not have the key. + debug!("Failed to get info about AUX files: {}", e); AuxFilesDirectory { files: HashMap::new(), } From c508d3b5fa73d7a301f561ac8854d49ed31813be Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 26 Oct 2023 12:36:38 +0300 Subject: [PATCH 23/69] reimpl Layer, remove remote layer, trait Layer, trait PersistentLayer (#4938) Implement a new `struct Layer` abstraction which manages downloadness internally, requiring no LayerMap locking or rewriting to download or evict providing a property "you have a layer, you can read it". The new `struct Layer` provides ability to keep the file resident via a RAII structure for new layers which still need to be uploaded. Previous solution solved this `RemoteTimelineClient::wait_completion` which lead to bugs like #5639. Evicting or the final local deletion after garbage collection is done using Arc'd value `Drop`. With a single `struct Layer` the closed open ended `trait Layer`, `trait PersistentLayer` and `struct RemoteLayer` are removed following noting that compaction could be simplified by simply not using any of the traits in between: #4839. The new `struct Layer` is a preliminary to remove `Timeline::layer_removal_cs` documented in #4745. Preliminaries: #4936, #4937, #5013, #5014, #5022, #5033, #5044, #5058, #5059, #5061, #5074, #5103, epic #5172, #5645, #5649. Related split off: #5057, #5134. --- libs/utils/src/lib.rs | 2 + libs/utils/src/sync.rs | 1 + libs/utils/src/sync/heavier_once_cell.rs | 306 ++++ pageserver/src/disk_usage_eviction_task.rs | 171 +- pageserver/src/http/routes.rs | 9 +- pageserver/src/metrics.rs | 2 +- pageserver/src/tenant.rs | 17 +- pageserver/src/tenant/layer_map.rs | 141 +- pageserver/src/tenant/par_fsync.rs | 3 +- .../src/tenant/remote_timeline_client.rs | 184 +- .../tenant/remote_timeline_client/upload.rs | 2 + pageserver/src/tenant/storage_layer.rs | 132 +- .../src/tenant/storage_layer/delta_layer.rs | 394 ++--- .../src/tenant/storage_layer/image_layer.rs | 272 +-- .../tenant/storage_layer/inmemory_layer.rs | 32 +- pageserver/src/tenant/storage_layer/layer.rs | 1481 +++++++++++++++++ .../src/tenant/storage_layer/layer_desc.rs | 59 +- .../src/tenant/storage_layer/remote_layer.rs | 216 --- pageserver/src/tenant/timeline.rs | 1391 +++++----------- .../src/tenant/timeline/eviction_task.rs | 42 +- .../src/tenant/timeline/layer_manager.rs | 177 +- pageserver/src/tenant/upload_queue.rs | 21 +- test_runner/fixtures/neon_fixtures.py | 2 +- .../regress/test_attach_tenant_config.py | 2 +- test_runner/regress/test_broken_timeline.py | 4 +- test_runner/regress/test_layer_eviction.py | 20 +- test_runner/regress/test_ondemand_download.py | 56 +- test_runner/regress/test_remote_storage.py | 2 +- .../test_tenants_with_remote_storage.py | 4 +- 29 files changed, 2740 insertions(+), 2405 deletions(-) create mode 100644 libs/utils/src/sync.rs create mode 100644 libs/utils/src/sync/heavier_once_cell.rs create mode 100644 pageserver/src/tenant/storage_layer/layer.rs delete mode 100644 pageserver/src/tenant/storage_layer/remote_layer.rs diff --git a/libs/utils/src/lib.rs b/libs/utils/src/lib.rs index 160e082833..b3e38cd9e1 100644 --- a/libs/utils/src/lib.rs +++ b/libs/utils/src/lib.rs @@ -73,6 +73,8 @@ pub mod completion; /// Reporting utilities pub mod error; +pub mod sync; + /// This is a shortcut to embed git sha into binaries and avoid copying the same build script to all packages /// /// we have several cases: diff --git a/libs/utils/src/sync.rs b/libs/utils/src/sync.rs new file mode 100644 index 0000000000..125eeca129 --- /dev/null +++ b/libs/utils/src/sync.rs @@ -0,0 +1 @@ +pub mod heavier_once_cell; diff --git a/libs/utils/src/sync/heavier_once_cell.rs b/libs/utils/src/sync/heavier_once_cell.rs new file mode 100644 index 0000000000..4d66a54c98 --- /dev/null +++ b/libs/utils/src/sync/heavier_once_cell.rs @@ -0,0 +1,306 @@ +use std::sync::{Arc, Mutex, MutexGuard}; +use tokio::sync::Semaphore; + +/// Custom design like [`tokio::sync::OnceCell`] but using [`OwnedSemaphorePermit`] instead of +/// `SemaphorePermit`, allowing use of `take` which does not require holding an outer mutex guard +/// for the duration of initialization. +/// +/// Has no unsafe, builds upon [`tokio::sync::Semaphore`] and [`std::sync::Mutex`]. +/// +/// [`OwnedSemaphorePermit`]: tokio::sync::OwnedSemaphorePermit +pub struct OnceCell { + inner: Mutex>, +} + +impl Default for OnceCell { + /// Create new uninitialized [`OnceCell`]. + fn default() -> Self { + Self { + inner: Default::default(), + } + } +} + +/// Semaphore is the current state: +/// - open semaphore means the value is `None`, not yet initialized +/// - closed semaphore means the value has been initialized +#[derive(Debug)] +struct Inner { + init_semaphore: Arc, + value: Option, +} + +impl Default for Inner { + fn default() -> Self { + Self { + init_semaphore: Arc::new(Semaphore::new(1)), + value: None, + } + } +} + +impl OnceCell { + /// Creates an already initialized `OnceCell` with the given value. + pub fn new(value: T) -> Self { + let sem = Semaphore::new(1); + sem.close(); + Self { + inner: Mutex::new(Inner { + init_semaphore: Arc::new(sem), + value: Some(value), + }), + } + } + + /// Returns a guard to an existing initialized value, or uniquely initializes the value before + /// returning the guard. + /// + /// Initializing might wait on any existing [`Guard::take_and_deinit`] deinitialization. + /// + /// Initialization is panic-safe and cancellation-safe. + pub async fn get_or_init(&self, factory: F) -> Result, E> + where + F: FnOnce() -> Fut, + Fut: std::future::Future>, + { + let sem = { + let guard = self.inner.lock().unwrap(); + if guard.value.is_some() { + return Ok(Guard(guard)); + } + guard.init_semaphore.clone() + }; + + let permit = sem.acquire_owned().await; + if permit.is_err() { + let guard = self.inner.lock().unwrap(); + assert!( + guard.value.is_some(), + "semaphore got closed, must be initialized" + ); + return Ok(Guard(guard)); + } else { + // now we try + let value = factory().await?; + + let mut guard = self.inner.lock().unwrap(); + assert!( + guard.value.is_none(), + "we won permit, must not be initialized" + ); + guard.value = Some(value); + guard.init_semaphore.close(); + Ok(Guard(guard)) + } + } + + /// Returns a guard to an existing initialized value, if any. + pub fn get(&self) -> Option> { + let guard = self.inner.lock().unwrap(); + if guard.value.is_some() { + Some(Guard(guard)) + } else { + None + } + } +} + +/// Uninteresting guard object to allow short-lived access to inspect or clone the held, +/// initialized value. +#[derive(Debug)] +pub struct Guard<'a, T>(MutexGuard<'a, Inner>); + +impl std::ops::Deref for Guard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.0 + .value + .as_ref() + .expect("guard is not created unless value has been initialized") + } +} + +impl std::ops::DerefMut for Guard<'_, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.0 + .value + .as_mut() + .expect("guard is not created unless value has been initialized") + } +} + +impl<'a, T> Guard<'a, T> { + /// Take the current value, and a new permit for it's deinitialization. + /// + /// The permit will be on a semaphore part of the new internal value, and any following + /// [`OnceCell::get_or_init`] will wait on it to complete. + pub fn take_and_deinit(&mut self) -> (T, tokio::sync::OwnedSemaphorePermit) { + let mut swapped = Inner::default(); + let permit = swapped + .init_semaphore + .clone() + .try_acquire_owned() + .expect("we just created this"); + std::mem::swap(&mut *self.0, &mut swapped); + swapped + .value + .map(|v| (v, permit)) + .expect("guard is not created unless value has been initialized") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{ + convert::Infallible, + sync::atomic::{AtomicUsize, Ordering}, + time::Duration, + }; + + #[tokio::test] + async fn many_initializers() { + #[derive(Default, Debug)] + struct Counters { + factory_got_to_run: AtomicUsize, + future_polled: AtomicUsize, + winners: AtomicUsize, + } + + let initializers = 100; + + let cell = Arc::new(OnceCell::default()); + let counters = Arc::new(Counters::default()); + let barrier = Arc::new(tokio::sync::Barrier::new(initializers + 1)); + + let mut js = tokio::task::JoinSet::new(); + for i in 0..initializers { + js.spawn({ + let cell = cell.clone(); + let counters = counters.clone(); + let barrier = barrier.clone(); + + async move { + barrier.wait().await; + let won = { + let g = cell + .get_or_init(|| { + counters.factory_got_to_run.fetch_add(1, Ordering::Relaxed); + async { + counters.future_polled.fetch_add(1, Ordering::Relaxed); + Ok::<_, Infallible>(i) + } + }) + .await + .unwrap(); + + *g == i + }; + + if won { + counters.winners.fetch_add(1, Ordering::Relaxed); + } + } + }); + } + + barrier.wait().await; + + while let Some(next) = js.join_next().await { + next.expect("no panics expected"); + } + + let mut counters = Arc::try_unwrap(counters).unwrap(); + + assert_eq!(*counters.factory_got_to_run.get_mut(), 1); + assert_eq!(*counters.future_polled.get_mut(), 1); + assert_eq!(*counters.winners.get_mut(), 1); + } + + #[tokio::test(start_paused = true)] + async fn reinit_waits_for_deinit() { + // with the tokio::time paused, we will "sleep" for 1s while holding the reinitialization + let sleep_for = Duration::from_secs(1); + let initial = 42; + let reinit = 1; + let cell = Arc::new(OnceCell::new(initial)); + + let deinitialization_started = Arc::new(tokio::sync::Barrier::new(2)); + + let jh = tokio::spawn({ + let cell = cell.clone(); + let deinitialization_started = deinitialization_started.clone(); + async move { + let (answer, _permit) = cell.get().expect("initialized to value").take_and_deinit(); + assert_eq!(answer, initial); + + deinitialization_started.wait().await; + tokio::time::sleep(sleep_for).await; + } + }); + + deinitialization_started.wait().await; + + let started_at = tokio::time::Instant::now(); + cell.get_or_init(|| async { Ok::<_, Infallible>(reinit) }) + .await + .unwrap(); + + let elapsed = started_at.elapsed(); + assert!( + elapsed >= sleep_for, + "initialization should had taken at least the time time slept with permit" + ); + + jh.await.unwrap(); + + assert_eq!(*cell.get().unwrap(), reinit); + } + + #[tokio::test] + async fn initialization_attemptable_until_ok() { + let cell = OnceCell::default(); + + for _ in 0..10 { + cell.get_or_init(|| async { Err("whatever error") }) + .await + .unwrap_err(); + } + + let g = cell + .get_or_init(|| async { Ok::<_, Infallible>("finally success") }) + .await + .unwrap(); + assert_eq!(*g, "finally success"); + } + + #[tokio::test] + async fn initialization_is_cancellation_safe() { + let cell = OnceCell::default(); + + let barrier = tokio::sync::Barrier::new(2); + + let initializer = cell.get_or_init(|| async { + barrier.wait().await; + futures::future::pending::<()>().await; + + Ok::<_, Infallible>("never reached") + }); + + tokio::select! { + _ = initializer => { unreachable!("cannot complete; stuck in pending().await") }, + _ = barrier.wait() => {} + }; + + // now initializer is dropped + + assert!(cell.get().is_none()); + + let g = cell + .get_or_init(|| async { Ok::<_, Infallible>("now initialized") }) + .await + .unwrap(); + assert_eq!(*g, "now initialized"); + } +} diff --git a/pageserver/src/disk_usage_eviction_task.rs b/pageserver/src/disk_usage_eviction_task.rs index b4d73baa88..413c941bc4 100644 --- a/pageserver/src/disk_usage_eviction_task.rs +++ b/pageserver/src/disk_usage_eviction_task.rs @@ -60,7 +60,11 @@ use utils::serde_percent::Percent; use crate::{ config::PageServerConf, task_mgr::{self, TaskKind, BACKGROUND_RUNTIME}, - tenant::{self, storage_layer::PersistentLayer, timeline::EvictionError, Timeline}, + tenant::{ + self, + storage_layer::{AsLayerDesc, EvictionError, Layer}, + Timeline, + }, }; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -108,7 +112,7 @@ pub fn launch_disk_usage_global_eviction_task( _ = background_jobs_barrier.wait() => { } }; - disk_usage_eviction_task(&state, task_config, storage, &conf.tenants_path(), cancel) + disk_usage_eviction_task(&state, task_config, &storage, &conf.tenants_path(), cancel) .await; Ok(()) }, @@ -121,7 +125,7 @@ pub fn launch_disk_usage_global_eviction_task( async fn disk_usage_eviction_task( state: &State, task_config: &DiskUsageEvictionTaskConfig, - storage: GenericRemoteStorage, + _storage: &GenericRemoteStorage, tenants_dir: &Utf8Path, cancel: CancellationToken, ) { @@ -145,14 +149,8 @@ async fn disk_usage_eviction_task( let start = Instant::now(); async { - let res = disk_usage_eviction_task_iteration( - state, - task_config, - &storage, - tenants_dir, - &cancel, - ) - .await; + let res = + disk_usage_eviction_task_iteration(state, task_config, tenants_dir, &cancel).await; match res { Ok(()) => {} @@ -183,13 +181,12 @@ pub trait Usage: Clone + Copy + std::fmt::Debug { async fn disk_usage_eviction_task_iteration( state: &State, task_config: &DiskUsageEvictionTaskConfig, - storage: &GenericRemoteStorage, tenants_dir: &Utf8Path, cancel: &CancellationToken, ) -> anyhow::Result<()> { let usage_pre = filesystem_level_usage::get(tenants_dir, task_config) .context("get filesystem-level disk usage before evictions")?; - let res = disk_usage_eviction_task_iteration_impl(state, storage, usage_pre, cancel).await; + let res = disk_usage_eviction_task_iteration_impl(state, usage_pre, cancel).await; match res { Ok(outcome) => { debug!(?outcome, "disk_usage_eviction_iteration finished"); @@ -273,7 +270,6 @@ struct LayerCount { pub async fn disk_usage_eviction_task_iteration_impl( state: &State, - storage: &GenericRemoteStorage, usage_pre: U, cancel: &CancellationToken, ) -> anyhow::Result> { @@ -330,9 +326,10 @@ pub async fn disk_usage_eviction_task_iteration_impl( // If we get far enough in the list that we start to evict layers that are below // the tenant's min-resident-size threshold, print a warning, and memorize the disk // usage at that point, in 'usage_planned_min_resident_size_respecting'. - let mut batched: HashMap<_, Vec>> = HashMap::new(); + let mut batched: HashMap<_, Vec<_>> = HashMap::new(); let mut warned = None; let mut usage_planned = usage_pre; + let mut max_batch_size = 0; for (i, (partition, candidate)) in candidates.into_iter().enumerate() { if !usage_planned.has_pressure() { debug!( @@ -349,10 +346,18 @@ pub async fn disk_usage_eviction_task_iteration_impl( usage_planned.add_available_bytes(candidate.layer.layer_desc().file_size); - batched - .entry(TimelineKey(candidate.timeline)) - .or_default() - .push(candidate.layer); + // FIXME: batching makes no sense anymore because of no layermap locking, should just spawn + // tasks to evict all seen layers until we have evicted enough + + let batch = batched.entry(TimelineKey(candidate.timeline)).or_default(); + + // semaphore will later be used to limit eviction concurrency, and we can express at + // most u32 number of permits. unlikely we would have u32::MAX layers to be evicted, + // but fail gracefully by not making batches larger. + if batch.len() < u32::MAX as usize { + batch.push(candidate.layer); + max_batch_size = max_batch_size.max(batch.len()); + } } let usage_planned = match warned { @@ -369,69 +374,101 @@ pub async fn disk_usage_eviction_task_iteration_impl( // phase2: evict victims batched by timeline - // After the loop, `usage_assumed` is the post-eviction usage, - // according to internal accounting. - let mut usage_assumed = usage_pre; - let mut evictions_failed = LayerCount::default(); + let mut js = tokio::task::JoinSet::new(); + + // ratelimit to 1k files or any higher max batch size + let limit = Arc::new(tokio::sync::Semaphore::new(1000.max(max_batch_size))); + for (timeline, batch) in batched { let tenant_id = timeline.tenant_id; let timeline_id = timeline.timeline_id; - let batch_size = batch.len(); + let batch_size = + u32::try_from(batch.len()).expect("batch size limited to u32::MAX during partitioning"); + + // I dislike naming of `available_permits` but it means current total amount of permits + // because permits can be added + assert!(batch_size as usize <= limit.available_permits()); debug!(%timeline_id, "evicting batch for timeline"); - async { - let results = timeline.evict_layers(storage, &batch, cancel.clone()).await; + let evict = { + let limit = limit.clone(); + let cancel = cancel.clone(); + async move { + let mut evicted_bytes = 0; + let mut evictions_failed = LayerCount::default(); - match results { - Err(e) => { - warn!("failed to evict batch: {:#}", e); - } - Ok(results) => { - assert_eq!(results.len(), batch.len()); - for (result, layer) in results.into_iter().zip(batch.iter()) { - let file_size = layer.layer_desc().file_size; - match result { - Some(Ok(())) => { - usage_assumed.add_available_bytes(file_size); - } - Some(Err(EvictionError::CannotEvictRemoteLayer)) => { - unreachable!("get_local_layers_for_disk_usage_eviction finds only local layers") - } - Some(Err(EvictionError::FileNotFound)) => { - evictions_failed.file_sizes += file_size; - evictions_failed.count += 1; - } - Some(Err( - e @ EvictionError::LayerNotFound(_) - | e @ EvictionError::StatFailed(_), - )) => { - let e = utils::error::report_compact_sources(&e); - warn!(%layer, "failed to evict layer: {e}"); - evictions_failed.file_sizes += file_size; - evictions_failed.count += 1; - } - Some(Err(EvictionError::MetadataInconsistency(detail))) => { - warn!(%layer, "failed to evict layer: {detail}"); - evictions_failed.file_sizes += file_size; - evictions_failed.count += 1; - } - None => { - assert!(cancel.is_cancelled()); - return; + let Ok(_permit) = limit.acquire_many_owned(batch_size).await else { + // semaphore closing means cancelled + return (evicted_bytes, evictions_failed); + }; + + let results = timeline.evict_layers(&batch, &cancel).await; + + match results { + Ok(results) => { + assert_eq!(results.len(), batch.len()); + for (result, layer) in results.into_iter().zip(batch.iter()) { + let file_size = layer.layer_desc().file_size; + match result { + Some(Ok(())) => { + evicted_bytes += file_size; + } + Some(Err(EvictionError::NotFound | EvictionError::Downloaded)) => { + evictions_failed.file_sizes += file_size; + evictions_failed.count += 1; + } + None => { + assert!(cancel.is_cancelled()); + } } } } + Err(e) => { + warn!("failed to evict batch: {:#}", e); + } } + (evicted_bytes, evictions_failed) } } - .instrument(tracing::info_span!("evict_batch", %tenant_id, %timeline_id, batch_size)) - .await; + .instrument(tracing::info_span!("evict_batch", %tenant_id, %timeline_id, batch_size)); - if cancel.is_cancelled() { + js.spawn(evict); + + // spwaning multiple thousands of these is essentially blocking, so give already spawned a + // chance of making progress + tokio::task::yield_now().await; + } + + let join_all = async move { + // After the evictions, `usage_assumed` is the post-eviction usage, + // according to internal accounting. + let mut usage_assumed = usage_pre; + let mut evictions_failed = LayerCount::default(); + + while let Some(res) = js.join_next().await { + match res { + Ok((evicted_bytes, failed)) => { + usage_assumed.add_available_bytes(evicted_bytes); + evictions_failed.file_sizes += failed.file_sizes; + evictions_failed.count += failed.count; + } + Err(je) if je.is_cancelled() => unreachable!("not used"), + Err(je) if je.is_panic() => { /* already logged */ } + Err(je) => tracing::error!("unknown JoinError: {je:?}"), + } + } + (usage_assumed, evictions_failed) + }; + + let (usage_assumed, evictions_failed) = tokio::select! { + tuple = join_all => { tuple }, + _ = cancel.cancelled() => { + // close the semaphore to stop any pending acquires + limit.close(); return Ok(IterationOutcome::Cancelled); } - } + }; Ok(IterationOutcome::Finished(IterationOutcomeFinished { before: usage_pre, @@ -446,7 +483,7 @@ pub async fn disk_usage_eviction_task_iteration_impl( #[derive(Clone)] struct EvictionCandidate { timeline: Arc, - layer: Arc, + layer: Layer, last_activity_ts: SystemTime, } diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 54c5cdf133..2c46d733d6 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -1205,7 +1205,7 @@ async fn timeline_compact_handler( timeline .compact(&cancel, &ctx) .await - .map_err(ApiError::InternalServerError)?; + .map_err(|e| ApiError::InternalServerError(e.into()))?; json_response(StatusCode::OK, ()) } .instrument(info_span!("manual_compaction", %tenant_id, %timeline_id)) @@ -1230,7 +1230,7 @@ async fn timeline_checkpoint_handler( timeline .compact(&cancel, &ctx) .await - .map_err(ApiError::InternalServerError)?; + .map_err(|e| ApiError::InternalServerError(e.into()))?; json_response(StatusCode::OK, ()) } @@ -1500,11 +1500,11 @@ async fn disk_usage_eviction_run( let state = get_state(&r); - let Some(storage) = state.remote_storage.clone() else { + if state.remote_storage.as_ref().is_none() { return Err(ApiError::InternalServerError(anyhow::anyhow!( "remote storage not configured, cannot run eviction iteration" ))); - }; + } let state = state.disk_usage_eviction_state.clone(); @@ -1522,7 +1522,6 @@ async fn disk_usage_eviction_run( async move { let res = crate::disk_usage_eviction_task::disk_usage_eviction_task_iteration_impl( &state, - &storage, usage, &child_cancel, ) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 267e632312..3e15be67dc 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1404,7 +1404,7 @@ impl TimelineMetrics { crate::metrics::RESIDENT_PHYSICAL_SIZE_GLOBAL.add(sz); } - pub fn resident_physical_size_get(&self) -> u64 { + pub(crate) fn resident_physical_size_get(&self) -> u64 { self.resident_physical_size_gauge.get() } } diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index f8895c32dc..f177a17afc 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -138,9 +138,7 @@ pub(crate) mod timeline; pub mod size; pub(crate) use timeline::span::debug_assert_current_span_has_tenant_and_timeline_id; -pub use timeline::{ - LocalLayerInfoForDiskUsageEviction, LogicalSizeCalculationCause, PageReconstructError, Timeline, -}; +pub(crate) use timeline::{LogicalSizeCalculationCause, PageReconstructError, Timeline}; // re-export for use in remote_timeline_client.rs pub use crate::tenant::metadata::save_metadata; @@ -4328,6 +4326,7 @@ mod tests { #[tokio::test] async fn delta_layer_dumping() -> anyhow::Result<()> { + use storage_layer::AsLayerDesc; let (tenant, ctx) = TenantHarness::create("test_layer_dumping")?.load().await; let tline = tenant .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) @@ -4335,16 +4334,18 @@ mod tests { make_some_layers(tline.as_ref(), Lsn(0x20), &ctx).await?; let layer_map = tline.layers.read().await; - let level0_deltas = layer_map.layer_map().get_level0_deltas()?; + let level0_deltas = layer_map + .layer_map() + .get_level0_deltas()? + .into_iter() + .map(|desc| layer_map.get_from_desc(&desc)) + .collect::>(); assert!(!level0_deltas.is_empty()); for delta in level0_deltas { - let delta = layer_map.get_from_desc(&delta); // Ensure we are dumping a delta layer here - let delta = delta.downcast_delta_layer().unwrap(); - - delta.dump(false, &ctx).await.unwrap(); + assert!(delta.layer_desc().is_delta); delta.dump(true, &ctx).await.unwrap(); } diff --git a/pageserver/src/tenant/layer_map.rs b/pageserver/src/tenant/layer_map.rs index b8185da010..9b6225501f 100644 --- a/pageserver/src/tenant/layer_map.rs +++ b/pageserver/src/tenant/layer_map.rs @@ -639,147 +639,10 @@ impl LayerMap { } println!("historic_layers:"); - for layer in self.iter_historic_layers() { - layer.dump(verbose, ctx)?; + for desc in self.iter_historic_layers() { + desc.dump(); } println!("End dump LayerMap"); Ok(()) } } - -#[cfg(test)] -mod tests { - use super::LayerMap; - use crate::tenant::storage_layer::LayerFileName; - use std::str::FromStr; - use std::sync::Arc; - - mod l0_delta_layers_updated { - - use crate::tenant::{ - storage_layer::{AsLayerDesc, PersistentLayerDesc}, - timeline::layer_manager::LayerFileManager, - }; - - use super::*; - - struct LayerObject(PersistentLayerDesc); - - impl AsLayerDesc for LayerObject { - fn layer_desc(&self) -> &PersistentLayerDesc { - &self.0 - } - } - - impl LayerObject { - fn new(desc: PersistentLayerDesc) -> Self { - LayerObject(desc) - } - } - - type TestLayerFileManager = LayerFileManager; - - #[test] - fn for_full_range_delta() { - // l0_delta_layers are used by compaction, and should observe all buffered updates - l0_delta_layers_updated_scenario( - "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000053423C21-0000000053424D69", - true - ) - } - - #[test] - fn for_non_full_range_delta() { - // has minimal uncovered areas compared to l0_delta_layers_updated_on_insert_replace_remove_for_full_range_delta - l0_delta_layers_updated_scenario( - "000000000000000000000000000000000001-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE__0000000053423C21-0000000053424D69", - // because not full range - false - ) - } - - #[test] - fn for_image() { - l0_delta_layers_updated_scenario( - "000000000000000000000000000000000000-000000000000000000000000000000010000__0000000053424D69", - // code only checks if it is a full range layer, doesn't care about images, which must - // mean we should in practice never have full range images - false - ) - } - - #[test] - fn replacing_missing_l0_is_notfound() { - // original impl had an oversight, and L0 was an anyhow::Error. anyhow::Error should - // however only happen for precondition failures. - - let layer = "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000053423C21-0000000053424D69"; - let layer = LayerFileName::from_str(layer).unwrap(); - let layer = PersistentLayerDesc::from(layer); - - // same skeletan construction; see scenario below - let not_found = Arc::new(LayerObject::new(layer.clone())); - let new_version = Arc::new(LayerObject::new(layer)); - - // after the immutable storage state refactor, the replace operation - // will not use layer map any more. We keep it here for consistency in test cases - // and can remove it in the future. - let _map = LayerMap::default(); - - let mut mapping = TestLayerFileManager::new(); - - mapping - .replace_and_verify(not_found, new_version) - .unwrap_err(); - } - - fn l0_delta_layers_updated_scenario(layer_name: &str, expected_l0: bool) { - let name = LayerFileName::from_str(layer_name).unwrap(); - let skeleton = PersistentLayerDesc::from(name); - - let remote = Arc::new(LayerObject::new(skeleton.clone())); - let downloaded = Arc::new(LayerObject::new(skeleton)); - - let mut map = LayerMap::default(); - let mut mapping = LayerFileManager::new(); - - // two disjoint Arcs in different lifecycle phases. even if it seems they must be the - // same layer, we use LayerMap::compare_arced_layers as the identity of layers. - assert_eq!(remote.layer_desc(), downloaded.layer_desc()); - - let expected_in_counts = (1, usize::from(expected_l0)); - - map.batch_update() - .insert_historic(remote.layer_desc().clone()); - mapping.insert(remote.clone()); - assert_eq!( - count_layer_in(&map, remote.layer_desc()), - expected_in_counts - ); - - mapping - .replace_and_verify(remote, downloaded.clone()) - .expect("name derived attributes are the same"); - assert_eq!( - count_layer_in(&map, downloaded.layer_desc()), - expected_in_counts - ); - - map.batch_update().remove_historic(downloaded.layer_desc()); - assert_eq!(count_layer_in(&map, downloaded.layer_desc()), (0, 0)); - } - - fn count_layer_in(map: &LayerMap, layer: &PersistentLayerDesc) -> (usize, usize) { - let historic = map - .iter_historic_layers() - .filter(|x| x.key() == layer.key()) - .count(); - let l0s = map - .get_level0_deltas() - .expect("why does this return a result"); - let l0 = l0s.iter().filter(|x| x.key() == layer.key()).count(); - - (historic, l0) - } - } -} diff --git a/pageserver/src/tenant/par_fsync.rs b/pageserver/src/tenant/par_fsync.rs index 3b1526e910..3acb0fb431 100644 --- a/pageserver/src/tenant/par_fsync.rs +++ b/pageserver/src/tenant/par_fsync.rs @@ -57,8 +57,7 @@ pub fn par_fsync(paths: &[Utf8PathBuf]) -> io::Result<()> { fsync_in_thread_pool(paths) } -/// Parallel fsync asynchronously. If number of files are less than PARALLEL_PATH_THRESHOLD, fsync is done in the current -/// execution thread. Otherwise, we will spawn_blocking and run it in tokio. +/// Parallel fsync asynchronously. pub async fn par_fsync_async(paths: &[Utf8PathBuf]) -> io::Result<()> { const MAX_CONCURRENT_FSYNC: usize = 64; let mut next = paths.iter().peekable(); diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index e8415ee9f3..245649c3ad 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -167,8 +167,6 @@ //! - download their remote [`IndexPart`]s //! - create `Timeline` struct and a `RemoteTimelineClient` //! - initialize the client's upload queue with its `IndexPart` -//! - create [`RemoteLayer`](super::storage_layer::RemoteLayer) instances -//! for layers that are referenced by `IndexPart` but not present locally //! - schedule uploads for layers that are only present locally. //! - if the remote `IndexPart`'s metadata was newer than the metadata in //! the local filesystem, write the remote metadata to the local filesystem @@ -211,8 +209,7 @@ mod upload; use anyhow::Context; use camino::Utf8Path; use chrono::{NaiveDateTime, Utc}; -// re-export these -pub use download::{is_temp_download_file, list_remote_timelines}; + use scopeguard::ScopeGuard; use tokio_util::sync::CancellationToken; use utils::backoff::{ @@ -237,7 +234,7 @@ use crate::metrics::{ }; use crate::task_mgr::shutdown_token; use crate::tenant::debug_assert_current_span_has_tenant_and_timeline_id; -use crate::tenant::remote_timeline_client::index::LayerFileMetadata; +use crate::tenant::storage_layer::AsLayerDesc; use crate::tenant::upload_queue::Delete; use crate::tenant::TIMELINES_SEGMENT_NAME; use crate::{ @@ -255,10 +252,13 @@ use utils::id::{TenantId, TimelineId}; use self::index::IndexPart; -use super::storage_layer::LayerFileName; +use super::storage_layer::{Layer, LayerFileName, ResidentLayer}; use super::upload_queue::SetDeletedFlagProgress; use super::Generation; +pub(crate) use download::{is_temp_download_file, list_remote_timelines}; +pub(crate) use index::LayerFileMetadata; + // Occasional network issues and such can cause remote operations to fail, and // that's expected. If a download fails, we log it at info-level, and retry. // But after FAILED_DOWNLOAD_WARN_THRESHOLD retries, we start to log it at WARN @@ -629,13 +629,12 @@ impl RemoteTimelineClient { /// pub(crate) fn schedule_layer_file_upload( self: &Arc, - layer_file_name: &LayerFileName, - layer_metadata: &LayerFileMetadata, + layer: ResidentLayer, ) -> anyhow::Result<()> { let mut guard = self.upload_queue.lock().unwrap(); let upload_queue = guard.initialized_mut()?; - self.schedule_layer_file_upload0(upload_queue, layer_file_name, layer_metadata); + self.schedule_layer_file_upload0(upload_queue, layer); self.launch_queued_tasks(upload_queue); Ok(()) } @@ -643,18 +642,19 @@ impl RemoteTimelineClient { fn schedule_layer_file_upload0( self: &Arc, upload_queue: &mut UploadQueueInitialized, - layer_file_name: &LayerFileName, - layer_metadata: &LayerFileMetadata, + layer: ResidentLayer, ) { + let metadata = layer.metadata(); + upload_queue .latest_files - .insert(layer_file_name.clone(), layer_metadata.clone()); + .insert(layer.layer_desc().filename(), metadata.clone()); upload_queue.latest_files_changes_since_metadata_upload_scheduled += 1; - let op = UploadOp::UploadLayer(layer_file_name.clone(), layer_metadata.clone()); + info!("scheduled layer file upload {layer}"); + let op = UploadOp::UploadLayer(layer, metadata); self.calls_unfinished_metric_begin(&op); upload_queue.queued_operations.push_back(op); - info!("scheduled layer file upload {layer_file_name}"); } /// Launch a delete operation in the background. @@ -667,13 +667,13 @@ impl RemoteTimelineClient { /// successfully. pub fn schedule_layer_file_deletion( self: &Arc, - names: Vec, + names: &[LayerFileName], ) -> anyhow::Result<()> { let mut guard = self.upload_queue.lock().unwrap(); let upload_queue = guard.initialized_mut()?; let with_generations = - self.schedule_unlinking_of_layers_from_index_part0(upload_queue, &names); + self.schedule_unlinking_of_layers_from_index_part0(upload_queue, names.iter().cloned()); self.schedule_deletion_of_unlinked0(upload_queue, with_generations); @@ -687,17 +687,17 @@ impl RemoteTimelineClient { /// /// The files will be leaked in remote storage unless [`Self::schedule_deletion_of_unlinked`] /// is invoked on them. - #[allow(unused)] // will be used by PR#4938 - pub(crate) fn schedule_unlinking_of_layers_from_index_part( - self: &Arc, - names: Vec, - ) -> anyhow::Result<()> { + pub(crate) fn schedule_gc_update(self: &Arc, gc_layers: &[Layer]) -> anyhow::Result<()> { let mut guard = self.upload_queue.lock().unwrap(); let upload_queue = guard.initialized_mut()?; // just forget the return value; after uploading the next index_part.json, we can consider - // the layer files as "dangling". this is fine however. - self.schedule_unlinking_of_layers_from_index_part0(upload_queue, &names); + // the layer files as "dangling". this is fine, at worst case we create work for the + // scrubber. + + let names = gc_layers.iter().map(|x| x.layer_desc().filename()); + + self.schedule_unlinking_of_layers_from_index_part0(upload_queue, names); self.launch_queued_tasks(upload_queue); @@ -706,26 +706,28 @@ impl RemoteTimelineClient { /// Update the remote index file, removing the to-be-deleted files from the index, /// allowing scheduling of actual deletions later. - fn schedule_unlinking_of_layers_from_index_part0( + fn schedule_unlinking_of_layers_from_index_part0( self: &Arc, upload_queue: &mut UploadQueueInitialized, - names: &[LayerFileName], - ) -> Vec<(LayerFileName, Generation)> { + names: I, + ) -> Vec<(LayerFileName, Generation)> + where + I: IntoIterator, + { // Deleting layers doesn't affect the values stored in TimelineMetadata, // so we don't need update it. Just serialize it. let metadata = upload_queue.latest_metadata.clone(); // Decorate our list of names with each name's generation, dropping - // makes that are unexpectedly missing from our metadata. + // names that are unexpectedly missing from our metadata. let with_generations: Vec<_> = names - .iter() + .into_iter() .filter_map(|name| { - // Remove from latest_files, learning the file's remote generation in the process - let meta = upload_queue.latest_files.remove(name); + let meta = upload_queue.latest_files.remove(&name); if let Some(meta) = meta { upload_queue.latest_files_changes_since_metadata_upload_scheduled += 1; - Some((name.to_owned(), meta.generation)) + Some((name, meta.generation)) } else { // This can only happen if we forgot to to schedule the file upload // before scheduling the delete. Log it because it is a rare/strange @@ -748,8 +750,7 @@ impl RemoteTimelineClient { } /// Schedules deletion for layer files which have previously been unlinked from the - /// `index_part.json` with [`Self::schedule_unlinking_of_layers_from_index_part`]. - #[allow(unused)] // will be used by Layer::drop in PR#4938 + /// `index_part.json` with [`Self::schedule_gc_update`] or [`Self::schedule_compaction_update`]. pub(crate) fn schedule_deletion_of_unlinked( self: &Arc, layers: Vec<(LayerFileName, Generation)>, @@ -784,18 +785,20 @@ impl RemoteTimelineClient { /// `compacted_from` represent the L0 names which have been `compacted_to` L1 layers. pub(crate) fn schedule_compaction_update( self: &Arc, - compacted_from: &[LayerFileName], - compacted_to: &[(LayerFileName, LayerFileMetadata)], + compacted_from: &[Layer], + compacted_to: &[ResidentLayer], ) -> anyhow::Result<()> { let mut guard = self.upload_queue.lock().unwrap(); let upload_queue = guard.initialized_mut()?; - for (name, m) in compacted_to { - self.schedule_layer_file_upload0(upload_queue, name, m); + for layer in compacted_to { + self.schedule_layer_file_upload0(upload_queue, layer.clone()); } + let names = compacted_from.iter().map(|x| x.layer_desc().filename()); + let with_generations = - self.schedule_unlinking_of_layers_from_index_part0(upload_queue, compacted_from); + self.schedule_unlinking_of_layers_from_index_part0(upload_queue, names); self.schedule_deletion_of_unlinked0(upload_queue, with_generations); self.launch_queued_tasks(upload_queue); @@ -1170,16 +1173,12 @@ impl RemoteTimelineClient { } let upload_result: anyhow::Result<()> = match &task.op { - UploadOp::UploadLayer(ref layer_file_name, ref layer_metadata) => { - let path = self - .conf - .timeline_path(&self.tenant_id, &self.timeline_id) - .join(layer_file_name.file_name()); - + UploadOp::UploadLayer(ref layer, ref layer_metadata) => { + let path = layer.local_path(); upload::upload_timeline_layer( self.conf, &self.storage_impl, - &path, + path, layer_metadata, self.generation, ) @@ -1496,13 +1495,6 @@ impl RemoteTimelineClient { } } } - - pub(crate) fn get_layer_metadata( - &self, - name: &LayerFileName, - ) -> anyhow::Result> { - self.upload_queue.lock().unwrap().get_layer_metadata(name) - } } pub fn remote_timelines_path(tenant_id: &TenantId) -> RemotePath { @@ -1590,6 +1582,7 @@ mod tests { context::RequestContext, tenant::{ harness::{TenantHarness, TIMELINE_ID}, + storage_layer::Layer, Generation, Tenant, Timeline, }, DEFAULT_PG_VERSION, @@ -1758,32 +1751,29 @@ mod tests { let generation = harness.generation; // Create a couple of dummy files, schedule upload for them - let layer_file_name_1: LayerFileName = "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(); - let layer_file_name_2: LayerFileName = "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D9-00000000016B5A52".parse().unwrap(); - let layer_file_name_3: LayerFileName = "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59DA-00000000016B5A53".parse().unwrap(); - let content_1 = dummy_contents("foo"); - let content_2 = dummy_contents("bar"); - let content_3 = dummy_contents("baz"); - for (filename, content) in [ - (&layer_file_name_1, &content_1), - (&layer_file_name_2, &content_2), - (&layer_file_name_3, &content_3), - ] { - std::fs::write(timeline_path.join(filename.file_name()), content).unwrap(); - } + let layers = [ + ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), dummy_contents("foo")), + ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D9-00000000016B5A52".parse().unwrap(), dummy_contents("bar")), + ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59DA-00000000016B5A53".parse().unwrap(), dummy_contents("baz")) + ] + .into_iter() + .map(|(name, contents): (LayerFileName, Vec)| { + std::fs::write(timeline_path.join(name.file_name()), &contents).unwrap(); + + Layer::for_resident( + harness.conf, + &timeline, + name, + LayerFileMetadata::new(contents.len() as u64, generation), + ) + }).collect::>(); client - .schedule_layer_file_upload( - &layer_file_name_1, - &LayerFileMetadata::new(content_1.len() as u64, generation), - ) + .schedule_layer_file_upload(layers[0].clone()) .unwrap(); client - .schedule_layer_file_upload( - &layer_file_name_2, - &LayerFileMetadata::new(content_2.len() as u64, generation), - ) + .schedule_layer_file_upload(layers[1].clone()) .unwrap(); // Check that they are started immediately, not queued @@ -1837,38 +1827,42 @@ mod tests { .collect(), &[ &initial_layer.file_name(), - &layer_file_name_1.file_name(), - &layer_file_name_2.file_name(), + &layers[0].layer_desc().filename().file_name(), + &layers[1].layer_desc().filename().file_name(), ], ); assert_eq!(index_part.metadata, metadata); // Schedule upload and then a deletion. Check that the deletion is queued client - .schedule_layer_file_upload( - &layer_file_name_3, - &LayerFileMetadata::new(content_3.len() as u64, generation), - ) + .schedule_layer_file_upload(layers[2].clone()) .unwrap(); + + // this is no longer consistent with how deletion works with Layer::drop, but in this test + // keep using schedule_layer_file_deletion because we don't have a way to wait for the + // spawn_blocking started by the drop. client - .schedule_layer_file_deletion([layer_file_name_1.clone()].to_vec()) + .schedule_layer_file_deletion(&[layers[0].layer_desc().filename()]) .unwrap(); { let mut guard = client.upload_queue.lock().unwrap(); let upload_queue = guard.initialized_mut().unwrap(); // Deletion schedules upload of the index file, and the file deletion itself - assert!(upload_queue.queued_operations.len() == 2); - assert!(upload_queue.inprogress_tasks.len() == 1); - assert!(upload_queue.num_inprogress_layer_uploads == 1); - assert!(upload_queue.num_inprogress_deletions == 0); - assert!(upload_queue.latest_files_changes_since_metadata_upload_scheduled == 0); + assert_eq!(upload_queue.queued_operations.len(), 2); + assert_eq!(upload_queue.inprogress_tasks.len(), 1); + assert_eq!(upload_queue.num_inprogress_layer_uploads, 1); + assert_eq!(upload_queue.num_inprogress_deletions, 0); + assert_eq!( + upload_queue.latest_files_changes_since_metadata_upload_scheduled, + 0 + ); } assert_remote_files( &[ &initial_layer.file_name(), - &layer_file_name_1.file_name(), - &layer_file_name_2.file_name(), + &layers[0].layer_desc().filename().file_name(), + &layers[1].layer_desc().filename().file_name(), "index_part.json", ], &remote_timeline_dir, @@ -1882,8 +1876,8 @@ mod tests { assert_remote_files( &[ &initial_layer.file_name(), - &layer_file_name_2.file_name(), - &layer_file_name_3.file_name(), + &layers[1].layer_desc().filename().file_name(), + &layers[2].layer_desc().filename().file_name(), "index_part.json", ], &remote_timeline_dir, @@ -1912,6 +1906,13 @@ mod tests { ) .unwrap(); + let layer_file_1 = Layer::for_resident( + harness.conf, + &timeline, + layer_file_name_1.clone(), + LayerFileMetadata::new(content_1.len() as u64, harness.generation), + ); + #[derive(Debug, PartialEq, Clone, Copy)] struct BytesStartedFinished { started: Option, @@ -1947,10 +1948,7 @@ mod tests { let actual_a = get_bytes_started_stopped(); client - .schedule_layer_file_upload( - &layer_file_name_1, - &LayerFileMetadata::new(content_1.len() as u64, harness.generation), - ) + .schedule_layer_file_upload(layer_file_1.clone()) .unwrap(); let actual_b = get_bytes_started_stopped(); diff --git a/pageserver/src/tenant/remote_timeline_client/upload.rs b/pageserver/src/tenant/remote_timeline_client/upload.rs index 03ba137566..0a37a8f283 100644 --- a/pageserver/src/tenant/remote_timeline_client/upload.rs +++ b/pageserver/src/tenant/remote_timeline_client/upload.rs @@ -72,6 +72,8 @@ pub(super) async fn upload_timeline_layer<'a>( // upload. However, a nonexistent file can also be indicative of // something worse, like when a file is scheduled for upload before // it has been written to disk yet. + // + // This is tested against `test_compaction_delete_before_upload` info!(path = %source_path, "File to upload doesn't exist. Likely the file has been deleted and an upload is not required any more."); return Ok(()); } diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index b3aacb20d2..3b2a61dcba 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -4,26 +4,21 @@ pub mod delta_layer; mod filename; mod image_layer; mod inmemory_layer; +mod layer; mod layer_desc; -mod remote_layer; -use crate::config::PageServerConf; use crate::context::{AccessStatsBehavior, RequestContext}; -use crate::repository::Key; use crate::task_mgr::TaskKind; use crate::walrecord::NeonWalRecord; -use anyhow::Result; use bytes::Bytes; -use camino::Utf8PathBuf; use enum_map::EnumMap; use enumset::EnumSet; use once_cell::sync::Lazy; -use pageserver_api::models::LayerAccessKind; use pageserver_api::models::{ - HistoricLayerInfo, LayerResidenceEvent, LayerResidenceEventReason, LayerResidenceStatus, + LayerAccessKind, LayerResidenceEvent, LayerResidenceEventReason, LayerResidenceStatus, }; use std::ops::Range; -use std::sync::{Arc, Mutex}; +use std::sync::Mutex; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tracing::warn; use utils::history_buffer::HistoryBufferWithDropCounter; @@ -39,7 +34,8 @@ pub use filename::{DeltaFileName, ImageFileName, LayerFileName}; pub use image_layer::{ImageLayer, ImageLayerWriter}; pub use inmemory_layer::InMemoryLayer; pub use layer_desc::{PersistentLayerDesc, PersistentLayerKey}; -pub use remote_layer::RemoteLayer; + +pub(crate) use layer::{EvictionError, Layer, ResidentLayer}; pub fn range_overlaps(a: &Range, b: &Range) -> bool where @@ -74,7 +70,7 @@ pub struct ValueReconstructState { pub img: Option<(Lsn, Bytes)>, } -/// Return value from Layer::get_page_reconstruct_data +/// Return value from [`Layer::get_value_reconstruct_data`] #[derive(Clone, Copy, Debug)] pub enum ValueReconstructResult { /// Got all the data needed to reconstruct the requested page @@ -179,26 +175,6 @@ impl LayerAccessStats { new } - /// Creates a clone of `self` and records `new_status` in the clone. - /// - /// The `new_status` is not recorded in `self`. - /// - /// See [`record_residence_event`] for why you need to do this while holding the layer map lock. - /// - /// [`record_residence_event`]: Self::record_residence_event - pub(crate) fn clone_for_residence_change( - &self, - new_status: LayerResidenceStatus, - ) -> LayerAccessStats { - let clone = { - let inner = self.0.lock().unwrap(); - inner.clone() - }; - let new = LayerAccessStats(Mutex::new(clone)); - new.record_residence_event(new_status, LayerResidenceEventReason::ResidenceChange); - new - } - /// Record a change in layer residency. /// /// Recording the event must happen while holding the layer map lock to @@ -321,95 +297,12 @@ impl LayerAccessStats { } } -/// Supertrait of the [`Layer`] trait that captures the bare minimum interface -/// required by [`LayerMap`](super::layer_map::LayerMap). -/// -/// All layers should implement a minimal `std::fmt::Debug` without tenant or -/// timeline names, because those are known in the context of which the layers -/// are used in (timeline). -#[async_trait::async_trait] -pub trait Layer: std::fmt::Debug + std::fmt::Display + Send + Sync + 'static { - /// - /// Return data needed to reconstruct given page at LSN. - /// - /// It is up to the caller to collect more data from previous layer and - /// perform WAL redo, if necessary. - /// - /// See PageReconstructResult for possible return values. The collected data - /// is appended to reconstruct_data; the caller should pass an empty struct - /// on first call, or a struct with a cached older image of the page if one - /// is available. If this returns ValueReconstructResult::Continue, look up - /// the predecessor layer and call again with the same 'reconstruct_data' to - /// collect more data. - async fn get_value_reconstruct_data( - &self, - key: Key, - lsn_range: Range, - reconstruct_data: &mut ValueReconstructState, - ctx: &RequestContext, - ) -> Result; -} - /// Get a layer descriptor from a layer. pub trait AsLayerDesc { /// Get the layer descriptor. fn layer_desc(&self) -> &PersistentLayerDesc; } -/// A Layer contains all data in a "rectangle" consisting of a range of keys and -/// range of LSNs. -/// -/// There are two kinds of layers, in-memory and on-disk layers. In-memory -/// layers are used to ingest incoming WAL, and provide fast access to the -/// recent page versions. On-disk layers are stored as files on disk, and are -/// immutable. This trait presents the common functionality of in-memory and -/// on-disk layers. -/// -/// Furthermore, there are two kinds of on-disk layers: delta and image layers. -/// A delta layer contains all modifications within a range of LSNs and keys. -/// An image layer is a snapshot of all the data in a key-range, at a single -/// LSN. -pub trait PersistentLayer: Layer + AsLayerDesc { - /// File name used for this layer, both in the pageserver's local filesystem - /// state as well as in the remote storage. - fn filename(&self) -> LayerFileName { - self.layer_desc().filename() - } - - // Path to the layer file in the local filesystem. - // `None` for `RemoteLayer`. - fn local_path(&self) -> Option; - - /// Permanently remove this layer from disk. - fn delete_resident_layer_file(&self) -> Result<()>; - - fn downcast_remote_layer(self: Arc) -> Option> { - None - } - - fn downcast_delta_layer(self: Arc) -> Option> { - None - } - - fn is_remote_layer(&self) -> bool { - false - } - - fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo; - - fn access_stats(&self) -> &LayerAccessStats; -} - -pub fn downcast_remote_layer( - layer: &Arc, -) -> Option> { - if layer.is_remote_layer() { - Arc::clone(layer).downcast_remote_layer() - } else { - None - } -} - pub mod tests { use super::*; @@ -447,19 +340,6 @@ pub mod tests { } } -/// Helper enum to hold a PageServerConf, or a path -/// -/// This is used by DeltaLayer and ImageLayer. Normally, this holds a reference to the -/// global config, and paths to layer files are constructed using the tenant/timeline -/// path from the config. But in the 'pagectl' binary, we need to construct a Layer -/// struct for a file on disk, without having a page server running, so that we have no -/// config. In that case, we use the Path variant to hold the full path to the file on -/// disk. -enum PathOrConf { - Path(Utf8PathBuf), - Conf(&'static PageServerConf), -} - /// Range wrapping newtype, which uses display to render Debug. /// /// Useful with `Key`, which has too verbose `{:?}` for printing multiple layers. diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 55fb491b65..4ccce3d6bd 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -34,18 +34,17 @@ use crate::repository::{Key, Value, KEY_SIZE}; use crate::tenant::blob_io::BlobWriter; use crate::tenant::block_io::{BlockBuf, BlockCursor, BlockLease, BlockReader, FileBlockReader}; use crate::tenant::disk_btree::{DiskBtreeBuilder, DiskBtreeReader, VisitDirection}; -use crate::tenant::storage_layer::{ - PersistentLayer, ValueReconstructResult, ValueReconstructState, -}; +use crate::tenant::storage_layer::{Layer, ValueReconstructResult, ValueReconstructState}; +use crate::tenant::Timeline; use crate::virtual_file::VirtualFile; use crate::{walrecord, TEMP_FILE_SUFFIX}; use crate::{DELTA_FILE_MAGIC, STORAGE_FORMAT_VERSION}; use anyhow::{bail, ensure, Context, Result}; use camino::{Utf8Path, Utf8PathBuf}; -use pageserver_api::models::{HistoricLayerInfo, LayerAccessKind}; +use pageserver_api::models::LayerAccessKind; use rand::{distributions::Alphanumeric, Rng}; use serde::{Deserialize, Serialize}; -use std::fs::{self, File}; +use std::fs::File; use std::io::SeekFrom; use std::ops::Range; use std::os::unix::fs::FileExt; @@ -59,10 +58,7 @@ use utils::{ lsn::Lsn, }; -use super::{ - AsLayerDesc, DeltaFileName, Layer, LayerAccessStats, LayerAccessStatsReset, PathOrConf, - PersistentLayerDesc, -}; +use super::{AsLayerDesc, LayerAccessStats, PersistentLayerDesc, ResidentLayer}; /// /// Header stored in the beginning of the file @@ -182,20 +178,12 @@ impl DeltaKey { } } -/// DeltaLayer is the in-memory data structure associated with an on-disk delta -/// file. -/// -/// We keep a DeltaLayer in memory for each file, in the LayerMap. If a layer -/// is in "loaded" state, we have a copy of the index in memory, in 'inner'. -/// Otherwise the struct is just a placeholder for a file that exists on disk, -/// and it needs to be loaded before using it in queries. +/// This is used only from `pagectl`. Within pageserver, all layers are +/// [`crate::tenant::storage_layer::Layer`], which can hold a [`DeltaLayerInner`]. pub struct DeltaLayer { - path_or_conf: PathOrConf, - + path: Utf8PathBuf, pub desc: PersistentLayerDesc, - access_stats: LayerAccessStats, - inner: OnceCell>, } @@ -212,6 +200,8 @@ impl std::fmt::Debug for DeltaLayer { } } +/// `DeltaLayerInner` is the in-memory data structure associated with an on-disk delta +/// file. pub struct DeltaLayerInner { // values copied from summary index_start_blk: u32, @@ -221,12 +211,6 @@ pub struct DeltaLayerInner { file: FileBlockReader, } -impl AsRef for DeltaLayerInner { - fn as_ref(&self) -> &DeltaLayerInner { - self - } -} - impl std::fmt::Debug for DeltaLayerInner { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("DeltaLayerInner") @@ -236,19 +220,6 @@ impl std::fmt::Debug for DeltaLayerInner { } } -#[async_trait::async_trait] -impl Layer for DeltaLayer { - async fn get_value_reconstruct_data( - &self, - key: Key, - lsn_range: Range, - reconstruct_state: &mut ValueReconstructState, - ctx: &RequestContext, - ) -> anyhow::Result { - self.get_value_reconstruct_data(key, lsn_range, reconstruct_state, ctx) - .await - } -} /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. impl std::fmt::Display for DeltaLayer { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -262,40 +233,9 @@ impl AsLayerDesc for DeltaLayer { } } -impl PersistentLayer for DeltaLayer { - fn downcast_delta_layer(self: Arc) -> Option> { - Some(self) - } - - fn local_path(&self) -> Option { - self.local_path() - } - - fn delete_resident_layer_file(&self) -> Result<()> { - self.delete_resident_layer_file() - } - - fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { - self.info(reset) - } - - fn access_stats(&self) -> &LayerAccessStats { - self.access_stats() - } -} - impl DeltaLayer { pub(crate) async fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()> { - println!( - "----- delta layer for ten {} tli {} keys {}-{} lsn {}-{} size {} ----", - self.desc.tenant_id, - self.desc.timeline_id, - self.desc.key_range.start, - self.desc.key_range.end, - self.desc.lsn_range.start, - self.desc.lsn_range.end, - self.desc.file_size, - ); + self.desc.dump(); if !verbose { return Ok(()); @@ -303,119 +243,7 @@ impl DeltaLayer { let inner = self.load(LayerAccessKind::Dump, ctx).await?; - println!( - "index_start_blk: {}, root {}", - inner.index_start_blk, inner.index_root_blk - ); - - let file = &inner.file; - let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new( - inner.index_start_blk, - inner.index_root_blk, - file, - ); - - tree_reader.dump().await?; - - let keys = DeltaLayerInner::load_keys(&inner, ctx).await?; - - // A subroutine to dump a single blob - async fn dump_blob(val: ValueRef<'_>, ctx: &RequestContext) -> Result { - let buf = val.reader.read_blob(val.blob_ref.pos(), ctx).await?; - let val = Value::des(&buf)?; - let desc = match val { - Value::Image(img) => { - format!(" img {} bytes", img.len()) - } - Value::WalRecord(rec) => { - let wal_desc = walrecord::describe_wal_record(&rec)?; - format!( - " rec {} bytes will_init: {} {}", - buf.len(), - rec.will_init(), - wal_desc - ) - } - }; - Ok(desc) - } - - for entry in keys { - let DeltaEntry { key, lsn, val, .. } = entry; - let desc = match dump_blob(val, ctx).await { - Ok(desc) => desc, - Err(err) => { - let err: anyhow::Error = err; - format!("ERROR: {err}") - } - }; - println!(" key {key} at {lsn}: {desc}"); - } - - Ok(()) - } - - pub(crate) async fn get_value_reconstruct_data( - &self, - key: Key, - lsn_range: Range, - reconstruct_state: &mut ValueReconstructState, - ctx: &RequestContext, - ) -> anyhow::Result { - ensure!(lsn_range.start >= self.desc.lsn_range.start); - - ensure!(self.desc.key_range.contains(&key)); - - let inner = self - .load(LayerAccessKind::GetValueReconstructData, ctx) - .await?; - inner - .get_value_reconstruct_data(key, lsn_range, reconstruct_state, ctx) - .await - } - - pub(crate) fn local_path(&self) -> Option { - Some(self.path()) - } - - pub(crate) fn delete_resident_layer_file(&self) -> Result<()> { - // delete underlying file - fs::remove_file(self.path())?; - Ok(()) - } - - pub(crate) fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { - let layer_file_name = self.layer_desc().filename().file_name(); - let lsn_range = self.layer_desc().lsn_range.clone(); - - let access_stats = self.access_stats.as_api_model(reset); - - HistoricLayerInfo::Delta { - layer_file_name, - layer_file_size: self.desc.file_size, - lsn_start: lsn_range.start, - lsn_end: lsn_range.end, - remote: false, - access_stats, - } - } - - pub(crate) fn access_stats(&self) -> &LayerAccessStats { - &self.access_stats - } - - fn path_for( - path_or_conf: &PathOrConf, - tenant_id: &TenantId, - timeline_id: &TimelineId, - fname: &DeltaFileName, - ) -> Utf8PathBuf { - match path_or_conf { - PathOrConf::Path(path) => path.clone(), - PathOrConf::Conf(conf) => conf - .timeline_path(tenant_id, timeline_id) - .join(fname.to_string()), - } + inner.dump(ctx).await } fn temp_path_for( @@ -461,52 +289,21 @@ impl DeltaLayer { async fn load_inner(&self, ctx: &RequestContext) -> Result> { let path = self.path(); - let summary = match &self.path_or_conf { - PathOrConf::Conf(_) => Some(Summary::from(self)), - PathOrConf::Path(_) => None, - }; + let loaded = DeltaLayerInner::load(&path, None, ctx).await?; - let loaded = DeltaLayerInner::load(&path, summary, ctx).await?; + // not production code + let actual_filename = path.file_name().unwrap().to_owned(); + let expected_filename = self.layer_desc().filename().file_name(); - if let PathOrConf::Path(ref path) = self.path_or_conf { - // not production code - - let actual_filename = path.file_name().unwrap().to_owned(); - let expected_filename = self.filename().file_name(); - - if actual_filename != expected_filename { - println!("warning: filename does not match what is expected from in-file summary"); - println!("actual: {:?}", actual_filename); - println!("expected: {:?}", expected_filename); - } + if actual_filename != expected_filename { + println!("warning: filename does not match what is expected from in-file summary"); + println!("actual: {:?}", actual_filename); + println!("expected: {:?}", expected_filename); } Ok(Arc::new(loaded)) } - /// Create a DeltaLayer struct representing an existing file on disk. - pub fn new( - conf: &'static PageServerConf, - timeline_id: TimelineId, - tenant_id: TenantId, - filename: &DeltaFileName, - file_size: u64, - access_stats: LayerAccessStats, - ) -> DeltaLayer { - DeltaLayer { - path_or_conf: PathOrConf::Conf(conf), - desc: PersistentLayerDesc::new_delta( - tenant_id, - timeline_id, - filename.key_range.clone(), - filename.lsn_range.clone(), - file_size, - ), - access_stats, - inner: OnceCell::new(), - } - } - /// Create a DeltaLayer struct representing an existing file on disk. /// /// This variant is only used for debugging purposes, by the 'pagectl' binary. @@ -520,7 +317,7 @@ impl DeltaLayer { .context("get file metadata to determine size")?; Ok(DeltaLayer { - path_or_conf: PathOrConf::Path(path.to_path_buf()), + path: path.to_path_buf(), desc: PersistentLayerDesc::new_delta( summary.tenant_id, summary.timeline_id, @@ -533,29 +330,9 @@ impl DeltaLayer { }) } - fn layer_name(&self) -> DeltaFileName { - self.desc.delta_file_name() - } /// Path to the layer file in pageserver workdir. - pub fn path(&self) -> Utf8PathBuf { - Self::path_for( - &self.path_or_conf, - &self.desc.tenant_id, - &self.desc.timeline_id, - &self.layer_name(), - ) - } - /// Loads all keys stored in the layer. Returns key, lsn, value size and value reference. - /// - /// The value can be obtained via the [`ValueRef::load`] function. - pub(crate) async fn load_keys(&self, ctx: &RequestContext) -> Result>> { - let inner = self - .load(LayerAccessKind::KeyIter, ctx) - .await - .context("load delta layer keys")?; - DeltaLayerInner::load_keys(inner, ctx) - .await - .context("Layer index is corrupted") + fn path(&self) -> Utf8PathBuf { + self.path.clone() } } @@ -660,7 +437,7 @@ impl DeltaLayerWriterInner { /// /// Finish writing the delta layer. /// - async fn finish(self, key_end: Key) -> anyhow::Result { + async fn finish(self, key_end: Key, timeline: &Arc) -> anyhow::Result { let index_start_blk = ((self.blob_writer.size() + PAGE_SZ as u64 - 1) / PAGE_SZ as u64) as u32; @@ -717,37 +494,21 @@ impl DeltaLayerWriterInner { // Note: Because we opened the file in write-only mode, we cannot // reuse the same VirtualFile for reading later. That's why we don't // set inner.file here. The first read will have to re-open it. - let layer = DeltaLayer { - path_or_conf: PathOrConf::Conf(self.conf), - desc: PersistentLayerDesc::new_delta( - self.tenant_id, - self.timeline_id, - self.key_start..key_end, - self.lsn_range.clone(), - metadata.len(), - ), - access_stats: LayerAccessStats::empty_will_record_residence_event_later(), - inner: OnceCell::new(), - }; + + let desc = PersistentLayerDesc::new_delta( + self.tenant_id, + self.timeline_id, + self.key_start..key_end, + self.lsn_range.clone(), + metadata.len(), + ); // fsync the file file.sync_all().await?; - // Rename the file to its final name - // - // Note: This overwrites any existing file. There shouldn't be any. - // FIXME: throw an error instead? - let final_path = DeltaLayer::path_for( - &PathOrConf::Conf(self.conf), - &self.tenant_id, - &self.timeline_id, - &DeltaFileName { - key_range: self.key_start..key_end, - lsn_range: self.lsn_range, - }, - ); - std::fs::rename(self.path, &final_path)?; - trace!("created delta layer {final_path}"); + let layer = Layer::finish_creating(self.conf, timeline, desc, &self.path)?; + + trace!("created delta layer {}", layer.local_path()); Ok(layer) } @@ -828,8 +589,12 @@ impl DeltaLayerWriter { /// /// Finish writing the delta layer. /// - pub async fn finish(mut self, key_end: Key) -> anyhow::Result { - self.inner.take().unwrap().finish(key_end).await + pub(crate) async fn finish( + mut self, + key_end: Key, + timeline: &Arc, + ) -> anyhow::Result { + self.inner.take().unwrap().finish(key_end, timeline).await } } @@ -967,15 +732,17 @@ impl DeltaLayerInner { } } - pub(super) async fn load_keys<'a, 'b, T: AsRef + Clone>( - this: &'a T, - ctx: &'b RequestContext, + pub(super) async fn load_keys<'a>( + &'a self, + ctx: &RequestContext, ) -> Result>> { - let dl = this.as_ref(); - let file = &dl.file; + let file = &self.file; - let tree_reader = - DiskBtreeReader::<_, DELTA_KEY_SIZE>::new(dl.index_start_blk, dl.index_root_blk, file); + let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new( + self.index_start_blk, + self.index_root_blk, + file, + ); let mut all_keys: Vec> = Vec::new(); @@ -988,7 +755,7 @@ impl DeltaLayerInner { let val_ref = ValueRef { blob_ref: BlobRef(value), reader: BlockCursor::new(crate::tenant::block_io::BlockReaderRef::Adapter( - Adapter(dl), + Adapter(self), )), }; let pos = BlobRef(value).pos(); @@ -1015,10 +782,61 @@ impl DeltaLayerInner { if let Some(last) = all_keys.last_mut() { // Last key occupies all space till end of value storage, // which corresponds to beginning of the index - last.size = dl.index_start_blk as u64 * PAGE_SZ as u64 - last.size; + last.size = self.index_start_blk as u64 * PAGE_SZ as u64 - last.size; } Ok(all_keys) } + + pub(super) async fn dump(&self, ctx: &RequestContext) -> anyhow::Result<()> { + println!( + "index_start_blk: {}, root {}", + self.index_start_blk, self.index_root_blk + ); + + let file = &self.file; + let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new( + self.index_start_blk, + self.index_root_blk, + file, + ); + + tree_reader.dump().await?; + + let keys = self.load_keys(ctx).await?; + + async fn dump_blob(val: ValueRef<'_>, ctx: &RequestContext) -> anyhow::Result { + let buf = val.reader.read_blob(val.blob_ref.pos(), ctx).await?; + let val = Value::des(&buf)?; + let desc = match val { + Value::Image(img) => { + format!(" img {} bytes", img.len()) + } + Value::WalRecord(rec) => { + let wal_desc = walrecord::describe_wal_record(&rec)?; + format!( + " rec {} bytes will_init: {} {}", + buf.len(), + rec.will_init(), + wal_desc + ) + } + }; + Ok(desc) + } + + for entry in keys { + let DeltaEntry { key, lsn, val, .. } = entry; + let desc = match dump_blob(val, ctx).await { + Ok(desc) => desc, + Err(err) => { + format!("ERROR: {err}") + } + }; + println!(" key {key} at {lsn}: {desc}"); + } + + Ok(()) + } } /// A set of data associated with a delta layer key and its value @@ -1058,3 +876,9 @@ impl> Adapter { self.0.as_ref().file.read_blk(blknum, ctx).await } } + +impl AsRef for DeltaLayerInner { + fn as_ref(&self) -> &DeltaLayerInner { + self + } +} diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index 94138a0786..b7b9ca69b0 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -31,21 +31,23 @@ use crate::tenant::blob_io::BlobWriter; use crate::tenant::block_io::{BlockBuf, BlockReader, FileBlockReader}; use crate::tenant::disk_btree::{DiskBtreeBuilder, DiskBtreeReader, VisitDirection}; use crate::tenant::storage_layer::{ - LayerAccessStats, PersistentLayer, ValueReconstructResult, ValueReconstructState, + LayerAccessStats, ValueReconstructResult, ValueReconstructState, }; +use crate::tenant::Timeline; use crate::virtual_file::VirtualFile; use crate::{IMAGE_FILE_MAGIC, STORAGE_FORMAT_VERSION, TEMP_FILE_SUFFIX}; use anyhow::{bail, ensure, Context, Result}; use bytes::Bytes; use camino::{Utf8Path, Utf8PathBuf}; use hex; -use pageserver_api::models::{HistoricLayerInfo, LayerAccessKind}; +use pageserver_api::models::LayerAccessKind; use rand::{distributions::Alphanumeric, Rng}; use serde::{Deserialize, Serialize}; -use std::fs::{self, File}; +use std::fs::File; use std::io::SeekFrom; use std::ops::Range; use std::os::unix::prelude::FileExt; +use std::sync::Arc; use tokio::sync::OnceCell; use tracing::*; @@ -56,7 +58,7 @@ use utils::{ }; use super::filename::ImageFileName; -use super::{AsLayerDesc, Layer, LayerAccessStatsReset, PathOrConf, PersistentLayerDesc}; +use super::{AsLayerDesc, Layer, PersistentLayerDesc, ResidentLayer}; /// /// Header stored in the beginning of the file @@ -114,22 +116,14 @@ impl Summary { } } -/// ImageLayer is the in-memory data structure associated with an on-disk image -/// file. -/// -/// We keep an ImageLayer in memory for each file, in the LayerMap. If a layer -/// is in "loaded" state, we have a copy of the index in memory, in 'inner'. -/// Otherwise the struct is just a placeholder for a file that exists on disk, -/// and it needs to be loaded before using it in queries. +/// This is used only from `pagectl`. Within pageserver, all layers are +/// [`crate::tenant::storage_layer::Layer`], which can hold an [`ImageLayerInner`]. pub struct ImageLayer { - path_or_conf: PathOrConf, - + path: Utf8PathBuf, pub desc: PersistentLayerDesc, // This entry contains an image of all pages as of this LSN, should be the same as desc.lsn pub lsn: Lsn, - access_stats: LayerAccessStats, - inner: OnceCell, } @@ -146,6 +140,8 @@ impl std::fmt::Debug for ImageLayer { } } +/// ImageLayer is the in-memory data structure associated with an on-disk image +/// file. pub struct ImageLayerInner { // values copied from summary index_start_blk: u32, @@ -166,73 +162,11 @@ impl std::fmt::Debug for ImageLayerInner { } } -#[async_trait::async_trait] -impl Layer for ImageLayer { - /// Look up given page in the file - async fn get_value_reconstruct_data( - &self, - key: Key, - lsn_range: Range, - reconstruct_state: &mut ValueReconstructState, - ctx: &RequestContext, - ) -> anyhow::Result { - self.get_value_reconstruct_data(key, lsn_range, reconstruct_state, ctx) - .await - } -} - -/// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. -impl std::fmt::Display for ImageLayer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.layer_desc().short_id()) - } -} - -impl AsLayerDesc for ImageLayer { - fn layer_desc(&self) -> &PersistentLayerDesc { - &self.desc - } -} - -impl PersistentLayer for ImageLayer { - fn local_path(&self) -> Option { - self.local_path() - } - - fn delete_resident_layer_file(&self) -> Result<()> { - self.delete_resident_layer_file() - } - - fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { - self.info(reset) - } - - fn access_stats(&self) -> &LayerAccessStats { - self.access_stats() - } -} - -impl ImageLayer { - pub(crate) async fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()> { - println!( - "----- image layer for ten {} tli {} key {}-{} at {} is_incremental {} size {} ----", - self.desc.tenant_id, - self.desc.timeline_id, - self.desc.key_range.start, - self.desc.key_range.end, - self.lsn, - self.desc.is_incremental(), - self.desc.file_size - ); - - if !verbose { - return Ok(()); - } - - let inner = self.load(LayerAccessKind::Dump, ctx).await?; - let file = &inner.file; +impl ImageLayerInner { + pub(super) async fn dump(&self, ctx: &RequestContext) -> anyhow::Result<()> { + let file = &self.file; let tree_reader = - DiskBtreeReader::<_, KEY_SIZE>::new(inner.index_start_blk, inner.index_root_blk, file); + DiskBtreeReader::<_, KEY_SIZE>::new(self.index_start_blk, self.index_root_blk, file); tree_reader.dump().await?; @@ -250,69 +184,36 @@ impl ImageLayer { Ok(()) } +} - pub(crate) async fn get_value_reconstruct_data( - &self, - key: Key, - lsn_range: Range, - reconstruct_state: &mut ValueReconstructState, - ctx: &RequestContext, - ) -> anyhow::Result { - assert!(self.desc.key_range.contains(&key)); - assert!(lsn_range.start >= self.lsn); - assert!(lsn_range.end >= self.lsn); - - let inner = self - .load(LayerAccessKind::GetValueReconstructData, ctx) - .await?; - inner - .get_value_reconstruct_data(key, reconstruct_state, ctx) - .await - // FIXME: makes no sense to dump paths - .with_context(|| format!("read {}", self.path())) +/// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. +impl std::fmt::Display for ImageLayer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.layer_desc().short_id()) } +} - pub(crate) fn local_path(&self) -> Option { - Some(self.path()) +impl AsLayerDesc for ImageLayer { + fn layer_desc(&self) -> &PersistentLayerDesc { + &self.desc } +} + +impl ImageLayer { + pub(crate) async fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()> { + self.desc.dump(); + + if !verbose { + return Ok(()); + } + + let inner = self.load(LayerAccessKind::Dump, ctx).await?; + + inner.dump(ctx).await?; - pub(crate) fn delete_resident_layer_file(&self) -> Result<()> { - // delete underlying file - fs::remove_file(self.path())?; Ok(()) } - pub(crate) fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { - let layer_file_name = self.layer_desc().filename().file_name(); - let lsn_start = self.layer_desc().image_layer_lsn(); - - HistoricLayerInfo::Image { - layer_file_name, - layer_file_size: self.desc.file_size, - lsn_start, - remote: false, - access_stats: self.access_stats.as_api_model(reset), - } - } - - pub(crate) fn access_stats(&self) -> &LayerAccessStats { - &self.access_stats - } - - fn path_for( - path_or_conf: &PathOrConf, - timeline_id: TimelineId, - tenant_id: TenantId, - fname: &ImageFileName, - ) -> Utf8PathBuf { - match path_or_conf { - PathOrConf::Path(path) => path.to_path_buf(), - PathOrConf::Conf(conf) => conf - .timeline_path(&tenant_id, &timeline_id) - .join(fname.to_string()), - } - } - fn temp_path_for( conf: &PageServerConf, timeline_id: TimelineId, @@ -348,54 +249,21 @@ impl ImageLayer { async fn load_inner(&self, ctx: &RequestContext) -> Result { let path = self.path(); - let expected_summary = match &self.path_or_conf { - PathOrConf::Conf(_) => Some(Summary::from(self)), - PathOrConf::Path(_) => None, - }; + let loaded = ImageLayerInner::load(&path, self.desc.image_layer_lsn(), None, ctx).await?; - let loaded = - ImageLayerInner::load(&path, self.desc.image_layer_lsn(), expected_summary, ctx) - .await?; + // not production code + let actual_filename = path.file_name().unwrap().to_owned(); + let expected_filename = self.layer_desc().filename().file_name(); - if let PathOrConf::Path(ref path) = self.path_or_conf { - // not production code - let actual_filename = path.file_name().unwrap().to_owned(); - let expected_filename = self.filename().file_name(); - - if actual_filename != expected_filename { - println!("warning: filename does not match what is expected from in-file summary"); - println!("actual: {:?}", actual_filename); - println!("expected: {:?}", expected_filename); - } + if actual_filename != expected_filename { + println!("warning: filename does not match what is expected from in-file summary"); + println!("actual: {:?}", actual_filename); + println!("expected: {:?}", expected_filename); } Ok(loaded) } - /// Create an ImageLayer struct representing an existing file on disk - pub fn new( - conf: &'static PageServerConf, - timeline_id: TimelineId, - tenant_id: TenantId, - filename: &ImageFileName, - file_size: u64, - access_stats: LayerAccessStats, - ) -> ImageLayer { - ImageLayer { - path_or_conf: PathOrConf::Conf(conf), - desc: PersistentLayerDesc::new_img( - tenant_id, - timeline_id, - filename.key_range.clone(), - filename.lsn, - file_size, - ), // Now we assume image layer ALWAYS covers the full range. This may change in the future. - lsn: filename.lsn, - access_stats, - inner: OnceCell::new(), - } - } - /// Create an ImageLayer struct representing an existing file on disk. /// /// This variant is only used for debugging purposes, by the 'pagectl' binary. @@ -407,7 +275,7 @@ impl ImageLayer { .metadata() .context("get file metadata to determine size")?; Ok(ImageLayer { - path_or_conf: PathOrConf::Path(path.to_path_buf()), + path: path.to_path_buf(), desc: PersistentLayerDesc::new_img( summary.tenant_id, summary.timeline_id, @@ -421,18 +289,8 @@ impl ImageLayer { }) } - fn layer_name(&self) -> ImageFileName { - self.desc.image_file_name() - } - - /// Path to the layer file in pageserver workdir. - pub fn path(&self) -> Utf8PathBuf { - Self::path_for( - &self.path_or_conf, - self.desc.timeline_id, - self.desc.tenant_id, - &self.layer_name(), - ) + fn path(&self) -> Utf8PathBuf { + self.path.clone() } } @@ -604,7 +462,7 @@ impl ImageLayerWriterInner { /// /// Finish writing the image layer. /// - async fn finish(self) -> anyhow::Result { + async fn finish(self, timeline: &Arc) -> anyhow::Result { let index_start_blk = ((self.blob_writer.size() + PAGE_SZ as u64 - 1) / PAGE_SZ as u64) as u32; @@ -658,33 +516,14 @@ impl ImageLayerWriterInner { // Note: Because we open the file in write-only mode, we cannot // reuse the same VirtualFile for reading later. That's why we don't // set inner.file here. The first read will have to re-open it. - let layer = ImageLayer { - path_or_conf: PathOrConf::Conf(self.conf), - desc, - lsn: self.lsn, - access_stats: LayerAccessStats::empty_will_record_residence_event_later(), - inner: OnceCell::new(), - }; // fsync the file file.sync_all().await?; - // Rename the file to its final name - // - // Note: This overwrites any existing file. There shouldn't be any. - // FIXME: throw an error instead? - let final_path = ImageLayer::path_for( - &PathOrConf::Conf(self.conf), - self.timeline_id, - self.tenant_id, - &ImageFileName { - key_range: self.key_range.clone(), - lsn: self.lsn, - }, - ); - std::fs::rename(self.path, final_path)?; + // FIXME: why not carry the virtualfile here, it supports renaming? + let layer = Layer::finish_creating(self.conf, timeline, desc, &self.path)?; - trace!("created image layer {}", layer.path()); + trace!("created image layer {}", layer.local_path()); Ok(layer) } @@ -746,8 +585,11 @@ impl ImageLayerWriter { /// /// Finish writing the image layer. /// - pub async fn finish(mut self) -> anyhow::Result { - self.inner.take().unwrap().finish().await + pub(crate) async fn finish( + mut self, + timeline: &Arc, + ) -> anyhow::Result { + self.inner.take().unwrap().finish(timeline).await } } diff --git a/pageserver/src/tenant/storage_layer/inmemory_layer.rs b/pageserver/src/tenant/storage_layer/inmemory_layer.rs index 764dc2c64e..1275250bf0 100644 --- a/pageserver/src/tenant/storage_layer/inmemory_layer.rs +++ b/pageserver/src/tenant/storage_layer/inmemory_layer.rs @@ -10,11 +10,12 @@ use crate::repository::{Key, Value}; use crate::tenant::block_io::BlockReader; use crate::tenant::ephemeral_file::EphemeralFile; use crate::tenant::storage_layer::{ValueReconstructResult, ValueReconstructState}; +use crate::tenant::Timeline; use crate::walrecord; use anyhow::{ensure, Result}; use pageserver_api::models::InMemoryLayerInfo; use std::collections::HashMap; -use std::sync::OnceLock; +use std::sync::{Arc, OnceLock}; use tracing::*; use utils::{ bin_ser::BeSer, @@ -28,7 +29,7 @@ use std::fmt::Write as _; use std::ops::Range; use tokio::sync::RwLock; -use super::{DeltaLayer, DeltaLayerWriter, Layer}; +use super::{DeltaLayerWriter, ResidentLayer}; pub struct InMemoryLayer { conf: &'static PageServerConf, @@ -207,20 +208,6 @@ impl InMemoryLayer { } } -#[async_trait::async_trait] -impl Layer for InMemoryLayer { - async fn get_value_reconstruct_data( - &self, - key: Key, - lsn_range: Range, - reconstruct_data: &mut ValueReconstructState, - ctx: &RequestContext, - ) -> Result { - self.get_value_reconstruct_data(key, lsn_range, reconstruct_data, ctx) - .await - } -} - impl std::fmt::Display for InMemoryLayer { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let end_lsn = self.end_lsn_or_max(); @@ -229,17 +216,13 @@ impl std::fmt::Display for InMemoryLayer { } impl InMemoryLayer { - /// /// Get layer size. - /// pub async fn size(&self) -> Result { let inner = self.inner.read().await; Ok(inner.file.len()) } - /// /// Create a new, empty, in-memory layer - /// pub async fn create( conf: &'static PageServerConf, timeline_id: TimelineId, @@ -331,7 +314,11 @@ impl InMemoryLayer { /// Write this frozen in-memory layer to disk. /// /// Returns a new delta layer with all the same data as this in-memory layer - pub(crate) async fn write_to_disk(&self, ctx: &RequestContext) -> Result { + pub(crate) async fn write_to_disk( + &self, + timeline: &Arc, + ctx: &RequestContext, + ) -> Result { // Grab the lock in read-mode. We hold it over the I/O, but because this // layer is not writeable anymore, no one should be trying to acquire the // write lock on it, so we shouldn't block anyone. There's one exception @@ -376,7 +363,8 @@ impl InMemoryLayer { } } - let delta_layer = delta_layer_writer.finish(Key::MAX).await?; + // MAX is used here because we identify L0 layers by full key range + let delta_layer = delta_layer_writer.finish(Key::MAX, timeline).await?; Ok(delta_layer) } } diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs new file mode 100644 index 0000000000..759f6d664c --- /dev/null +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -0,0 +1,1481 @@ +use anyhow::Context; +use camino::{Utf8Path, Utf8PathBuf}; +use pageserver_api::models::{ + HistoricLayerInfo, LayerAccessKind, LayerResidenceEventReason, LayerResidenceStatus, +}; +use std::ops::Range; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Weak}; +use std::time::SystemTime; +use tracing::Instrument; +use utils::lsn::Lsn; +use utils::sync::heavier_once_cell; + +use crate::config::PageServerConf; +use crate::context::RequestContext; +use crate::repository::Key; +use crate::tenant::{remote_timeline_client::LayerFileMetadata, RemoteTimelineClient, Timeline}; + +use super::delta_layer::{self, DeltaEntry}; +use super::image_layer; +use super::{ + AsLayerDesc, LayerAccessStats, LayerAccessStatsReset, LayerFileName, PersistentLayerDesc, + ValueReconstructResult, ValueReconstructState, +}; + +use utils::generation::Generation; + +/// A Layer contains all data in a "rectangle" consisting of a range of keys and +/// range of LSNs. +/// +/// There are two kinds of layers, in-memory and on-disk layers. In-memory +/// layers are used to ingest incoming WAL, and provide fast access to the +/// recent page versions. On-disk layers are stored as files on disk, and are +/// immutable. This type represents the on-disk kind while in-memory kind are represented by +/// [`InMemoryLayer`]. +/// +/// Furthermore, there are two kinds of on-disk layers: delta and image layers. +/// A delta layer contains all modifications within a range of LSNs and keys. +/// An image layer is a snapshot of all the data in a key-range, at a single +/// LSN. +/// +/// This type models the on-disk layers, which can be evicted and on-demand downloaded. +/// +/// [`InMemoryLayer`]: super::inmemory_layer::InMemoryLayer +#[derive(Clone)] +pub(crate) struct Layer(Arc); + +impl std::fmt::Display for Layer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.layer_desc().short_id()) + } +} + +impl std::fmt::Debug for Layer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self) + } +} + +impl AsLayerDesc for Layer { + fn layer_desc(&self) -> &PersistentLayerDesc { + self.0.layer_desc() + } +} + +impl Layer { + /// Creates a layer value for a file we know to not be resident. + pub(crate) fn for_evicted( + conf: &'static PageServerConf, + timeline: &Arc, + file_name: LayerFileName, + metadata: LayerFileMetadata, + ) -> Self { + let desc = PersistentLayerDesc::from_filename( + timeline.tenant_id, + timeline.timeline_id, + file_name, + metadata.file_size(), + ); + + let access_stats = LayerAccessStats::for_loading_layer(LayerResidenceStatus::Evicted); + + let owner = Layer(Arc::new(LayerInner::new( + conf, + timeline, + access_stats, + desc, + None, + metadata.generation, + ))); + + debug_assert!(owner.0.needs_download_blocking().unwrap().is_some()); + + owner + } + + /// Creates a Layer value for a file we know to be resident in timeline directory. + pub(crate) fn for_resident( + conf: &'static PageServerConf, + timeline: &Arc, + file_name: LayerFileName, + metadata: LayerFileMetadata, + ) -> ResidentLayer { + let desc = PersistentLayerDesc::from_filename( + timeline.tenant_id, + timeline.timeline_id, + file_name, + metadata.file_size(), + ); + + let access_stats = LayerAccessStats::for_loading_layer(LayerResidenceStatus::Resident); + + let mut resident = None; + + let owner = Layer(Arc::new_cyclic(|owner| { + let inner = Arc::new(DownloadedLayer { + owner: owner.clone(), + kind: tokio::sync::OnceCell::default(), + }); + resident = Some(inner.clone()); + + LayerInner::new( + conf, + timeline, + access_stats, + desc, + Some(inner), + metadata.generation, + ) + })); + + let downloaded = resident.expect("just initialized"); + + debug_assert!(owner.0.needs_download_blocking().unwrap().is_none()); + + timeline + .metrics + .resident_physical_size_add(metadata.file_size()); + + ResidentLayer { downloaded, owner } + } + + /// Creates a Layer value for freshly written out new layer file by renaming it from a + /// temporary path. + pub(crate) fn finish_creating( + conf: &'static PageServerConf, + timeline: &Arc, + desc: PersistentLayerDesc, + temp_path: &Utf8Path, + ) -> anyhow::Result { + let mut resident = None; + + let owner = Layer(Arc::new_cyclic(|owner| { + let inner = Arc::new(DownloadedLayer { + owner: owner.clone(), + kind: tokio::sync::OnceCell::default(), + }); + resident = Some(inner.clone()); + let access_stats = LayerAccessStats::empty_will_record_residence_event_later(); + access_stats.record_residence_event( + LayerResidenceStatus::Resident, + LayerResidenceEventReason::LayerCreate, + ); + LayerInner::new( + conf, + timeline, + access_stats, + desc, + Some(inner), + timeline.generation, + ) + })); + + let downloaded = resident.expect("just initialized"); + + // if the rename works, the path is as expected + std::fs::rename(temp_path, owner.local_path()) + .with_context(|| format!("rename temporary file as correct path for {owner}"))?; + + Ok(ResidentLayer { downloaded, owner }) + } + + /// Requests the layer to be evicted and waits for this to be done. + /// + /// If the file is not resident, an [`EvictionError::NotFound`] is returned. + /// + /// If for a bad luck or blocking of the executor, we miss the actual eviction and the layer is + /// re-downloaded, [`EvictionError::Downloaded`] is returned. + /// + /// Technically cancellation safe, but cancelling might shift the viewpoint of what generation + /// of download-evict cycle on retry. + pub(crate) async fn evict_and_wait( + &self, + rtc: &RemoteTimelineClient, + ) -> Result<(), EvictionError> { + self.0.evict_and_wait(rtc).await + } + + /// Delete the layer file when the `self` gets dropped, also try to schedule a remote index upload + /// then. + /// + /// On drop, this will cause a call to [`RemoteTimelineClient::schedule_deletion_of_unlinked`]. + /// This means that the unlinking by [gc] or [compaction] must have happened strictly before + /// the value this is called on gets dropped. + /// + /// This is ensured by both of those methods accepting references to Layer. + /// + /// [gc]: [`RemoteTimelineClient::schedule_gc_update`] + /// [compaction]: [`RemoteTimelineClient::schedule_compaction_update`] + pub(crate) fn garbage_collect_on_drop(&self) { + self.0.garbage_collect_on_drop(); + } + + /// Return data needed to reconstruct given page at LSN. + /// + /// It is up to the caller to collect more data from the previous layer and + /// perform WAL redo, if necessary. + pub(crate) async fn get_value_reconstruct_data( + &self, + key: Key, + lsn_range: Range, + reconstruct_data: &mut ValueReconstructState, + ctx: &RequestContext, + ) -> anyhow::Result { + use anyhow::ensure; + + let layer = self.0.get_or_maybe_download(true, Some(ctx)).await?; + self.0 + .access_stats + .record_access(LayerAccessKind::GetValueReconstructData, ctx); + + if self.layer_desc().is_delta { + ensure!(lsn_range.start >= self.layer_desc().lsn_range.start); + ensure!(self.layer_desc().key_range.contains(&key)); + } else { + ensure!(self.layer_desc().key_range.contains(&key)); + ensure!(lsn_range.start >= self.layer_desc().image_layer_lsn()); + ensure!(lsn_range.end >= self.layer_desc().image_layer_lsn()); + } + + layer + .get_value_reconstruct_data(key, lsn_range, reconstruct_data, &self.0, ctx) + .await + } + + /// Download the layer if evicted. + /// + /// Will not error when the layer is already downloaded. + pub(crate) async fn download(&self) -> anyhow::Result<()> { + self.0.get_or_maybe_download(true, None).await?; + Ok(()) + } + + /// Assuming the layer is already downloaded, returns a guard which will prohibit eviction + /// while the guard exists. + /// + /// Returns None if the layer is currently evicted. + pub(crate) async fn keep_resident(&self) -> anyhow::Result> { + let downloaded = match self.0.get_or_maybe_download(false, None).await { + Ok(d) => d, + // technically there are a lot of possible errors, but in practice it should only be + // DownloadRequired which is tripped up. could work to improve this situation + // statically later. + Err(DownloadError::DownloadRequired) => return Ok(None), + Err(e) => return Err(e.into()), + }; + + Ok(Some(ResidentLayer { + downloaded, + owner: self.clone(), + })) + } + + /// Downloads if necessary and creates a guard, which will keep this layer from being evicted. + pub(crate) async fn download_and_keep_resident(&self) -> anyhow::Result { + let downloaded = self.0.get_or_maybe_download(true, None).await?; + + Ok(ResidentLayer { + downloaded, + owner: self.clone(), + }) + } + + pub(crate) fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { + self.0.info(reset) + } + + pub(crate) fn access_stats(&self) -> &LayerAccessStats { + &self.0.access_stats + } + + pub(crate) fn local_path(&self) -> &Utf8Path { + &self.0.path + } + + pub(crate) fn metadata(&self) -> LayerFileMetadata { + self.0.metadata() + } + + /// Traditional debug dumping facility + #[allow(unused)] + pub(crate) async fn dump(&self, verbose: bool, ctx: &RequestContext) -> anyhow::Result<()> { + self.0.desc.dump(); + + if verbose { + // for now, unconditionally download everything, even if that might not be wanted. + let l = self.0.get_or_maybe_download(true, Some(ctx)).await?; + l.dump(&self.0, ctx).await? + } + + Ok(()) + } +} + +/// The download-ness ([`DownloadedLayer`]) can be either resident or wanted evicted. +/// +/// However when we want something evicted, we cannot evict it right away as there might be current +/// reads happening on it. For example: it has been searched from [`LayerMap::search`] but not yet +/// read with [`Layer::get_value_reconstruct_data`]. +/// +/// [`LayerMap::search`]: crate::tenant::layer_map::LayerMap::search +enum ResidentOrWantedEvicted { + Resident(Arc), + WantedEvicted(Weak), +} + +impl ResidentOrWantedEvicted { + fn get(&self) -> Option> { + match self { + ResidentOrWantedEvicted::Resident(strong) => Some(strong.clone()), + ResidentOrWantedEvicted::WantedEvicted(weak) => match weak.upgrade() { + Some(strong) => { + LAYER_IMPL_METRICS.inc_raced_wanted_evicted_accesses(); + Some(strong) + } + None => None, + }, + } + } + /// When eviction is first requested, drop down to holding a [`Weak`]. + /// + /// Returns `true` if this was the first time eviction was requested. + fn downgrade(&mut self) -> &Weak { + let _was_first = match self { + ResidentOrWantedEvicted::Resident(strong) => { + let weak = Arc::downgrade(strong); + *self = ResidentOrWantedEvicted::WantedEvicted(weak); + // returning the weak is not useful, because the drop could had already ran with + // the replacement above, and that will take care of cleaning the Option we are in + true + } + ResidentOrWantedEvicted::WantedEvicted(_) => false, + }; + + match self { + ResidentOrWantedEvicted::WantedEvicted(ref weak) => weak, + _ => unreachable!("just wrote wanted evicted"), + } + } +} + +struct LayerInner { + /// Only needed to check ondemand_download_behavior_treat_error_as_warn and creation of + /// [`Self::path`]. + conf: &'static PageServerConf, + + /// Full path to the file; unclear if this should exist anymore. + path: Utf8PathBuf, + + desc: PersistentLayerDesc, + + /// Timeline access is needed for remote timeline client and metrics. + timeline: Weak, + + /// Cached knowledge of [`Timeline::remote_client`] being `Some`. + have_remote_client: bool, + + access_stats: LayerAccessStats, + + /// This custom OnceCell is backed by std mutex, but only held for short time periods. + /// Initialization and deinitialization are done while holding a permit. + inner: heavier_once_cell::OnceCell, + + /// Do we want to garbage collect this when `LayerInner` is dropped + wanted_garbage_collected: AtomicBool, + + /// Do we want to evict this layer as soon as possible? After being set to `true`, all accesses + /// will try to downgrade [`ResidentOrWantedEvicted`], which will eventually trigger + /// [`LayerInner::on_downloaded_layer_drop`]. + wanted_evicted: AtomicBool, + + /// Version is to make sure we will in fact only evict a file if no new download has been + /// started. + version: AtomicUsize, + + /// Allow subscribing to when the layer actually gets evicted. + status: tokio::sync::broadcast::Sender, + + /// Counter for exponential backoff with the download + consecutive_failures: AtomicUsize, + + /// The generation of this Layer. + /// + /// For loaded layers (resident or evicted) this comes from [`LayerFileMetadata::generation`], + /// for created layers from [`Timeline::generation`]. + generation: Generation, +} + +impl std::fmt::Display for LayerInner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.layer_desc().short_id()) + } +} + +impl AsLayerDesc for LayerInner { + fn layer_desc(&self) -> &PersistentLayerDesc { + &self.desc + } +} + +#[derive(Debug, Clone, Copy)] +enum Status { + Evicted, + Downloaded, +} + +impl Drop for LayerInner { + fn drop(&mut self) { + if !*self.wanted_garbage_collected.get_mut() { + // should we try to evict if the last wish was for eviction? + // feels like there's some hazard of overcrowding near shutdown near by, but we don't + // run drops during shutdown (yet) + return; + } + + let span = tracing::info_span!(parent: None, "layer_gc", tenant_id = %self.layer_desc().tenant_id, timeline_id = %self.layer_desc().timeline_id, layer = %self); + + let path = std::mem::take(&mut self.path); + let file_name = self.layer_desc().filename(); + let gen = self.generation; + let file_size = self.layer_desc().file_size; + let timeline = self.timeline.clone(); + + crate::task_mgr::BACKGROUND_RUNTIME.spawn_blocking(move || { + let _g = span.entered(); + + let removed = match std::fs::remove_file(path) { + Ok(()) => true, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + // until we no longer do detaches by removing all local files before removing the + // tenant from the global map, we will always get these errors even if we knew what + // is the latest state. + // + // we currently do not track the latest state, so we'll also end up here on evicted + // layers. + false + } + Err(e) => { + tracing::error!("failed to remove garbage collected layer: {e}"); + LAYER_IMPL_METRICS.inc_gc_removes_failed(); + false + } + }; + + if let Some(timeline) = timeline.upgrade() { + if removed { + timeline.metrics.resident_physical_size_sub(file_size); + } + if let Some(remote_client) = timeline.remote_client.as_ref() { + let res = remote_client.schedule_deletion_of_unlinked(vec![(file_name, gen)]); + + if let Err(e) = res { + // test_timeline_deletion_with_files_stuck_in_upload_queue is good at + // demonstrating this deadlock (without spawn_blocking): stop will drop + // queued items, which will have ResidentLayer's, and those drops would try + // to re-entrantly lock the RemoteTimelineClient inner state. + if !timeline.is_active() { + tracing::info!("scheduling deletion on drop failed: {e:#}"); + } else { + tracing::warn!("scheduling deletion on drop failed: {e:#}"); + } + LAYER_IMPL_METRICS.inc_gcs_failed(GcFailed::DeleteSchedulingFailed); + } else { + LAYER_IMPL_METRICS.inc_completed_gcs(); + } + } + } else { + // no need to nag that timeline is gone: under normal situation on + // task_mgr::remove_tenant_from_memory the timeline is gone before we get dropped. + LAYER_IMPL_METRICS.inc_gcs_failed(GcFailed::TimelineGone); + } + }); + } +} + +impl LayerInner { + fn new( + conf: &'static PageServerConf, + timeline: &Arc, + access_stats: LayerAccessStats, + desc: PersistentLayerDesc, + downloaded: Option>, + generation: Generation, + ) -> Self { + let path = conf + .timeline_path(&timeline.tenant_id, &timeline.timeline_id) + .join(desc.filename().to_string()); + + LayerInner { + conf, + path, + desc, + timeline: Arc::downgrade(timeline), + have_remote_client: timeline.remote_client.is_some(), + access_stats, + wanted_garbage_collected: AtomicBool::new(false), + wanted_evicted: AtomicBool::new(false), + inner: if let Some(inner) = downloaded { + heavier_once_cell::OnceCell::new(ResidentOrWantedEvicted::Resident(inner)) + } else { + heavier_once_cell::OnceCell::default() + }, + version: AtomicUsize::new(0), + status: tokio::sync::broadcast::channel(1).0, + consecutive_failures: AtomicUsize::new(0), + generation, + } + } + + fn garbage_collect_on_drop(&self) { + let res = self.wanted_garbage_collected.compare_exchange( + false, + true, + Ordering::Release, + Ordering::Relaxed, + ); + + if res.is_ok() { + LAYER_IMPL_METRICS.inc_started_gcs(); + } + } + + pub(crate) async fn evict_and_wait( + &self, + _: &RemoteTimelineClient, + ) -> Result<(), EvictionError> { + use tokio::sync::broadcast::error::RecvError; + + assert!(self.have_remote_client); + + let mut rx = self.status.subscribe(); + + let res = + self.wanted_evicted + .compare_exchange(false, true, Ordering::Release, Ordering::Relaxed); + + if res.is_ok() { + LAYER_IMPL_METRICS.inc_started_evictions(); + } + + if self.get().is_none() { + // it was not evictable in the first place + // our store to the wanted_evicted does not matter; it will be reset by next download + return Err(EvictionError::NotFound); + } + + match rx.recv().await { + Ok(Status::Evicted) => Ok(()), + Ok(Status::Downloaded) => Err(EvictionError::Downloaded), + Err(RecvError::Closed) => { + unreachable!("sender cannot be dropped while we are in &self method") + } + Err(RecvError::Lagged(_)) => { + // this is quite unlikely, but we are blocking a lot in the async context, so + // we might be missing this because we are stuck on a LIFO slot on a thread + // which is busy blocking for a 1TB database create_image_layers. + // + // use however late (compared to the initial expressing of wanted) as the + // "outcome" now + match self.get() { + Some(_) => Err(EvictionError::Downloaded), + None => Ok(()), + } + } + } + } + + /// Should be cancellation safe, but cancellation is troublesome together with the spawned + /// download. + async fn get_or_maybe_download( + self: &Arc, + allow_download: bool, + ctx: Option<&RequestContext>, + ) -> Result, DownloadError> { + loop { + let download = move || async move { + // disable any scheduled but not yet running eviction deletions for this + self.version.fetch_add(1, Ordering::Relaxed); + + // no need to make the evict_and_wait wait for the actual download to complete + drop(self.status.send(Status::Downloaded)); + + let timeline = self + .timeline + .upgrade() + .ok_or_else(|| DownloadError::TimelineShutdown)?; + + let can_ever_evict = timeline.remote_client.as_ref().is_some(); + + // check if we really need to be downloaded; could have been already downloaded by a + // cancelled previous attempt. + let needs_download = self + .needs_download() + .await + .map_err(DownloadError::PreStatFailed)?; + + if let Some(reason) = needs_download { + // only reset this after we've decided we really need to download. otherwise it'd + // be impossible to mark cancelled downloads for eviction, like one could imagine + // we would like to do for prefetching which was not needed. + self.wanted_evicted.store(false, Ordering::Release); + + if !can_ever_evict { + return Err(DownloadError::NoRemoteStorage); + } + + tracing::debug!(%reason, "downloading layer"); + + if let Some(ctx) = ctx { + self.check_expected_download(ctx)?; + } + + if !allow_download { + // this does look weird, but for LayerInner the "downloading" means also changing + // internal once related state ... + return Err(DownloadError::DownloadRequired); + } + + self.spawn_download_and_wait(timeline).await?; + } else { + // the file is present locally, probably by a previous but cancelled call to + // get_or_maybe_download. alternatively we might be running without remote storage. + LAYER_IMPL_METRICS.inc_init_needed_no_download(); + } + + let res = Arc::new(DownloadedLayer { + owner: Arc::downgrade(self), + kind: tokio::sync::OnceCell::default(), + }); + + self.access_stats.record_residence_event( + LayerResidenceStatus::Resident, + LayerResidenceEventReason::ResidenceChange, + ); + + Ok(ResidentOrWantedEvicted::Resident(res)) + }; + + let locked = self.inner.get_or_init(download).await?; + + if let Some(strong) = Self::get_or_apply_evictedness(Some(locked), &self.wanted_evicted) + { + return Ok(strong); + } + + // the situation in which we might need to retry is that our init was ready + // immediatedly, but the DownloadedLayer had been dropped BUT failed to complete + // Self::evict_blocking + LAYER_IMPL_METRICS.inc_retried_get_or_maybe_download(); + } + } + + /// Nag or fail per RequestContext policy + fn check_expected_download(&self, ctx: &RequestContext) -> Result<(), DownloadError> { + use crate::context::DownloadBehavior::*; + let b = ctx.download_behavior(); + match b { + Download => Ok(()), + Warn | Error => { + tracing::warn!( + "unexpectedly on-demand downloading remote layer {self} for task kind {:?}", + ctx.task_kind() + ); + crate::metrics::UNEXPECTED_ONDEMAND_DOWNLOADS.inc(); + + let really_error = + matches!(b, Error) && !self.conf.ondemand_download_behavior_treat_error_as_warn; + + if really_error { + // this check is only probablistic, seems like flakyness footgun + Err(DownloadError::ContextAndConfigReallyDeniesDownloads) + } else { + Ok(()) + } + } + } + } + + /// Actual download, at most one is executed at the time. + async fn spawn_download_and_wait( + self: &Arc, + timeline: Arc, + ) -> Result<(), DownloadError> { + let task_name = format!("download layer {}", self); + + let (tx, rx) = tokio::sync::oneshot::channel(); + // this is sadly needed because of task_mgr::shutdown_tasks, otherwise we cannot + // block tenant::mgr::remove_tenant_from_memory. + + let this: Arc = self.clone(); + crate::task_mgr::spawn( + &tokio::runtime::Handle::current(), + crate::task_mgr::TaskKind::RemoteDownloadTask, + Some(self.desc.tenant_id), + Some(self.desc.timeline_id), + &task_name, + false, + async move { + let client = timeline + .remote_client + .as_ref() + .expect("checked above with have_remote_client"); + + let result = client.download_layer_file( + &this.desc.filename(), + &this.metadata(), + ) + .await; + + let result = match result { + Ok(size) => { + timeline.metrics.resident_physical_size_add(size); + Ok(()) + } + Err(e) => { + Err(e) + } + }; + + if let Err(res) = tx.send(result) { + match res { + Ok(()) => { + // our caller is cancellation safe so this is fine; if someone + // else requests the layer, they'll find it already downloaded + // or redownload. + // + // however, could be that we should consider marking the layer + // for eviction? alas, cannot: because only DownloadedLayer + // will handle that. + tracing::info!("layer file download completed after requester had cancelled"); + LAYER_IMPL_METRICS.inc_download_completed_without_requester(); + }, + Err(e) => { + // our caller is cancellation safe, but we might be racing with + // another attempt to initialize. before we have cancellation + // token support: these attempts should converge regardless of + // their completion order. + tracing::error!("layer file download failed, and additionally failed to communicate this to caller: {e:?}"); + LAYER_IMPL_METRICS.inc_download_failed_without_requester(); + } + } + } + + Ok(()) + } + .in_current_span(), + ); + match rx.await { + Ok(Ok(())) => { + if let Some(reason) = self + .needs_download() + .await + .map_err(DownloadError::PostStatFailed)? + { + // this is really a bug in needs_download or remote timeline client + panic!("post-condition failed: needs_download returned {reason:?}"); + } + + self.consecutive_failures.store(0, Ordering::Relaxed); + + Ok(()) + } + Ok(Err(e)) => { + let consecutive_failures = + self.consecutive_failures.fetch_add(1, Ordering::Relaxed); + tracing::error!(consecutive_failures, "layer file download failed: {e:#}"); + let backoff = utils::backoff::exponential_backoff_duration_seconds( + consecutive_failures.min(u32::MAX as usize) as u32, + 1.5, + 60.0, + ); + let backoff = std::time::Duration::from_secs_f64(backoff); + + tokio::time::sleep(backoff).await; + Err(DownloadError::DownloadFailed) + } + Err(_gone) => Err(DownloadError::DownloadCancelled), + } + } + + /// Access the current state without waiting for the file to be downloaded. + /// + /// Requires that we've initialized to state which is respective to the + /// actual residency state. + fn get(&self) -> Option> { + let locked = self.inner.get(); + Self::get_or_apply_evictedness(locked, &self.wanted_evicted) + } + + fn get_or_apply_evictedness( + guard: Option>, + wanted_evicted: &AtomicBool, + ) -> Option> { + if let Some(mut x) = guard { + if let Some(won) = x.get() { + // there are no guarantees that we will always get to observe a concurrent call + // to evict + if wanted_evicted.load(Ordering::Acquire) { + x.downgrade(); + } + return Some(won); + } + } + + None + } + + async fn needs_download(&self) -> Result, std::io::Error> { + match tokio::fs::metadata(&self.path).await { + Ok(m) => Ok(self.is_file_present_and_good_size(&m).err()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Some(NeedsDownload::NotFound)), + Err(e) => Err(e), + } + } + + fn needs_download_blocking(&self) -> Result, std::io::Error> { + match self.path.metadata() { + Ok(m) => Ok(self.is_file_present_and_good_size(&m).err()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Some(NeedsDownload::NotFound)), + Err(e) => Err(e), + } + } + + fn is_file_present_and_good_size(&self, m: &std::fs::Metadata) -> Result<(), NeedsDownload> { + // in future, this should include sha2-256 validation of the file. + if !m.is_file() { + Err(NeedsDownload::NotFile) + } else if m.len() != self.desc.file_size { + Err(NeedsDownload::WrongSize { + actual: m.len(), + expected: self.desc.file_size, + }) + } else { + Ok(()) + } + } + + fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { + let layer_file_name = self.desc.filename().file_name(); + + let remote = self.get().is_none(); + + let access_stats = self.access_stats.as_api_model(reset); + + if self.desc.is_delta { + let lsn_range = &self.desc.lsn_range; + + HistoricLayerInfo::Delta { + layer_file_name, + layer_file_size: self.desc.file_size, + lsn_start: lsn_range.start, + lsn_end: lsn_range.end, + remote, + access_stats, + } + } else { + let lsn = self.desc.image_layer_lsn(); + + HistoricLayerInfo::Image { + layer_file_name, + layer_file_size: self.desc.file_size, + lsn_start: lsn, + remote, + access_stats, + } + } + } + + /// `DownloadedLayer` is being dropped, so it calls this method. + fn on_downloaded_layer_drop(self: Arc) { + let gc = self.wanted_garbage_collected.load(Ordering::Acquire); + let evict = self.wanted_evicted.load(Ordering::Acquire); + let can_evict = self.have_remote_client; + + if gc { + // do nothing now, only in LayerInner::drop + } else if can_evict && evict { + let version = self.version.load(Ordering::Relaxed); + + let span = tracing::info_span!(parent: None, "layer_evict", tenant_id = %self.desc.tenant_id, timeline_id = %self.desc.timeline_id, layer=%self); + + // downgrade for queueing, in case there's a tear down already ongoing we should not + // hold it alive. + let this = Arc::downgrade(&self); + drop(self); + + crate::task_mgr::BACKGROUND_RUNTIME.spawn_blocking(move || { + let _g = span.entered(); + + // if LayerInner is already dropped here, do nothing because the garbage collection + // has already ran while we were in queue + let Some(this) = this.upgrade() else { + LAYER_IMPL_METRICS.inc_eviction_cancelled(EvictionCancelled::LayerGone); + return; + }; + this.evict_blocking(version); + }); + } + } + + fn evict_blocking(&self, version: usize) { + match self.evict_blocking0(version) { + Ok(()) => LAYER_IMPL_METRICS.inc_completed_evictions(), + Err(reason) => LAYER_IMPL_METRICS.inc_eviction_cancelled(reason), + } + } + + fn evict_blocking0(&self, version: usize) -> Result<(), EvictionCancelled> { + // deleted or detached timeline, don't do anything. + let Some(timeline) = self.timeline.upgrade() else { + return Err(EvictionCancelled::TimelineGone); + }; + + // to avoid starting a new download while we evict, keep holding on to the + // permit. + let _permit = { + let maybe_downloaded = self.inner.get(); + + if version != self.version.load(Ordering::Relaxed) { + // downloadness-state has advanced, we might no longer be the latest eviction + // work; don't do anything. + // + // this is possible to get to by having: + // + // 1. wanted_evicted.store(true) + // 2. ResidentOrWantedEvicted::downgrade + // 3. DownloadedLayer::drop + // 4. LayerInner::get_or_maybe_download + // 5. LayerInner::evict_blocking + return Err(EvictionCancelled::VersionCheckFailed); + } + + // free the DownloadedLayer allocation + match maybe_downloaded.map(|mut g| g.take_and_deinit()) { + Some((taken, permit)) => { + assert!(matches!(taken, ResidentOrWantedEvicted::WantedEvicted(_))); + permit + } + None => { + unreachable!("we do the version checking for this exact reason") + } + } + }; + + self.access_stats.record_residence_event( + LayerResidenceStatus::Evicted, + LayerResidenceEventReason::ResidenceChange, + ); + + let res = match capture_mtime_and_remove(&self.path) { + Ok(local_layer_mtime) => { + let duration = SystemTime::now().duration_since(local_layer_mtime); + match duration { + Ok(elapsed) => { + timeline + .metrics + .evictions_with_low_residence_duration + .read() + .unwrap() + .observe(elapsed); + tracing::info!( + residence_millis = elapsed.as_millis(), + "evicted layer after known residence period" + ); + } + Err(_) => { + tracing::info!("evicted layer after unknown residence period"); + } + } + timeline + .metrics + .resident_physical_size_sub(self.desc.file_size); + + Ok(()) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + tracing::info!("failed to evict file from disk, it was already gone"); + Err(EvictionCancelled::FileNotFound) + } + Err(e) => { + tracing::warn!("failed to evict file from disk: {e:#}"); + Err(EvictionCancelled::RemoveFailed) + } + }; + + // we are still holding the permit, so no new spawn_download_and_wait can happen + drop(self.status.send(Status::Evicted)); + + res + } + + fn metadata(&self) -> LayerFileMetadata { + LayerFileMetadata::new(self.desc.file_size, self.generation) + } +} + +fn capture_mtime_and_remove(path: &Utf8Path) -> Result { + let m = path.metadata()?; + let local_layer_mtime = m.modified()?; + std::fs::remove_file(path)?; + Ok(local_layer_mtime) +} + +#[derive(Debug, thiserror::Error)] +pub(crate) enum EvictionError { + #[error("layer was already evicted")] + NotFound, + + /// Evictions must always lose to downloads in races, and this time it happened. + #[error("layer was downloaded instead")] + Downloaded, +} + +/// Error internal to the [`LayerInner::get_or_maybe_download`] +#[derive(Debug, thiserror::Error)] +enum DownloadError { + #[error("timeline has already shutdown")] + TimelineShutdown, + #[error("no remote storage configured")] + NoRemoteStorage, + #[error("context denies downloading")] + ContextAndConfigReallyDeniesDownloads, + #[error("downloading is really required but not allowed by this method")] + DownloadRequired, + /// Why no error here? Because it will be reported by page_service. We should had also done + /// retries already. + #[error("downloading evicted layer file failed")] + DownloadFailed, + #[error("downloading failed, possibly for shutdown")] + DownloadCancelled, + #[error("pre-condition: stat before download failed")] + PreStatFailed(#[source] std::io::Error), + #[error("post-condition: stat after download failed")] + PostStatFailed(#[source] std::io::Error), +} + +#[derive(Debug, PartialEq)] +pub(crate) enum NeedsDownload { + NotFound, + NotFile, + WrongSize { actual: u64, expected: u64 }, +} + +impl std::fmt::Display for NeedsDownload { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + NeedsDownload::NotFound => write!(f, "file was not found"), + NeedsDownload::NotFile => write!(f, "path is not a file"), + NeedsDownload::WrongSize { actual, expected } => { + write!(f, "file size mismatch {actual} vs. {expected}") + } + } + } +} + +/// Existence of `DownloadedLayer` means that we have the file locally, and can later evict it. +pub(crate) struct DownloadedLayer { + owner: Weak, + kind: tokio::sync::OnceCell>, +} + +impl std::fmt::Debug for DownloadedLayer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DownloadedLayer") + // owner omitted because it is always "Weak" + .field("kind", &self.kind) + .finish() + } +} + +impl Drop for DownloadedLayer { + fn drop(&mut self) { + if let Some(owner) = self.owner.upgrade() { + owner.on_downloaded_layer_drop(); + } else { + // no need to do anything, we are shutting down + } + } +} + +impl DownloadedLayer { + /// Initializes the `DeltaLayerInner` or `ImageLayerInner` within [`LayerKind`], or fails to + /// initialize it permanently. + /// + /// `owner` parameter is a strong reference at the same `LayerInner` as the + /// `DownloadedLayer::owner` would be when upgraded. Given how this method ends up called, + /// we will always have the LayerInner on the callstack, so we can just use it. + async fn get<'a>( + &'a self, + owner: &Arc, + ctx: &RequestContext, + ) -> anyhow::Result<&'a LayerKind> { + let init = || async { + assert_eq!( + Weak::as_ptr(&self.owner), + Arc::as_ptr(owner), + "these are the same, just avoiding the upgrade" + ); + + // there is nothing async here, but it should be async + let res = if owner.desc.is_delta { + let summary = Some(delta_layer::Summary::expected( + owner.desc.tenant_id, + owner.desc.timeline_id, + owner.desc.key_range.clone(), + owner.desc.lsn_range.clone(), + )); + delta_layer::DeltaLayerInner::load(&owner.path, summary, ctx) + .await + .map(LayerKind::Delta) + } else { + let lsn = owner.desc.image_layer_lsn(); + let summary = Some(image_layer::Summary::expected( + owner.desc.tenant_id, + owner.desc.timeline_id, + owner.desc.key_range.clone(), + lsn, + )); + image_layer::ImageLayerInner::load(&owner.path, lsn, summary, ctx) + .await + .map(LayerKind::Image) + } + // this will be a permanent failure + .context("load layer"); + + if res.is_err() { + LAYER_IMPL_METRICS.inc_permanent_loading_failures(); + } + res + }; + self.kind.get_or_init(init).await.as_ref().map_err(|e| { + // errors are not clonabled, cannot but stringify + // test_broken_timeline matches this string + anyhow::anyhow!("layer loading failed: {e:#}") + }) + } + + async fn get_value_reconstruct_data( + &self, + key: Key, + lsn_range: Range, + reconstruct_data: &mut ValueReconstructState, + owner: &Arc, + ctx: &RequestContext, + ) -> anyhow::Result { + use LayerKind::*; + + match self.get(owner, ctx).await? { + Delta(d) => { + d.get_value_reconstruct_data(key, lsn_range, reconstruct_data, ctx) + .await + } + Image(i) => { + i.get_value_reconstruct_data(key, reconstruct_data, ctx) + .await + } + } + } + + async fn dump(&self, owner: &Arc, ctx: &RequestContext) -> anyhow::Result<()> { + use LayerKind::*; + match self.get(owner, ctx).await? { + Delta(d) => d.dump(ctx).await?, + Image(i) => i.dump(ctx).await?, + } + + Ok(()) + } +} + +/// Wrapper around an actual layer implementation. +#[derive(Debug)] +enum LayerKind { + Delta(delta_layer::DeltaLayerInner), + Image(image_layer::ImageLayerInner), +} + +/// Guard for forcing a layer be resident while it exists. +#[derive(Clone)] +pub(crate) struct ResidentLayer { + owner: Layer, + downloaded: Arc, +} + +impl std::fmt::Display for ResidentLayer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.owner) + } +} + +impl std::fmt::Debug for ResidentLayer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.owner) + } +} + +impl ResidentLayer { + /// Release the eviction guard, converting back into a plain [`Layer`]. + pub(crate) fn drop_eviction_guard(self) -> Layer { + self.into() + } + + /// Loads all keys stored in the layer. Returns key, lsn and value size. + pub(crate) async fn load_keys<'a>( + &'a self, + ctx: &RequestContext, + ) -> anyhow::Result>> { + use LayerKind::*; + + let owner = &self.owner.0; + + match self.downloaded.get(owner, ctx).await? { + Delta(ref d) => { + owner + .access_stats + .record_access(LayerAccessKind::KeyIter, ctx); + + // this is valid because the DownloadedLayer::kind is a OnceCell, not a + // Mutex, so we cannot go and deinitialize the value with OnceCell::take + // while it's being held. + delta_layer::DeltaLayerInner::load_keys(d, ctx) + .await + .context("Layer index is corrupted") + } + Image(_) => anyhow::bail!("cannot load_keys on a image layer"), + } + } + + pub(crate) fn local_path(&self) -> &Utf8Path { + &self.owner.0.path + } + + pub(crate) fn access_stats(&self) -> &LayerAccessStats { + self.owner.access_stats() + } + + pub(crate) fn metadata(&self) -> LayerFileMetadata { + self.owner.metadata() + } +} + +impl AsLayerDesc for ResidentLayer { + fn layer_desc(&self) -> &PersistentLayerDesc { + self.owner.layer_desc() + } +} + +impl AsRef for ResidentLayer { + fn as_ref(&self) -> &Layer { + &self.owner + } +} + +/// Allow slimming down if we don't want the `2*usize` with eviction candidates? +impl From for Layer { + fn from(value: ResidentLayer) -> Self { + value.owner + } +} + +use metrics::{IntCounter, IntCounterVec}; + +struct LayerImplMetrics { + started_evictions: IntCounter, + completed_evictions: IntCounter, + cancelled_evictions: IntCounterVec, + + started_gcs: IntCounter, + completed_gcs: IntCounter, + failed_gcs: IntCounterVec, + + rare_counters: IntCounterVec, +} + +impl Default for LayerImplMetrics { + fn default() -> Self { + let evictions = metrics::register_int_counter_vec!( + "pageserver_layer_evictions_count", + "Evictions started and completed in the Layer implementation", + &["state"] + ) + .unwrap(); + + let started_evictions = evictions + .get_metric_with_label_values(&["started"]) + .unwrap(); + let completed_evictions = evictions + .get_metric_with_label_values(&["completed"]) + .unwrap(); + + let cancelled_evictions = metrics::register_int_counter_vec!( + "pageserver_layer_cancelled_evictions_count", + "Different reasons for evictions to have been cancelled or failed", + &["reason"] + ) + .unwrap(); + + let gcs = metrics::register_int_counter_vec!( + "pageserver_layer_gcs_count", + "Garbage collections started and completed in the Layer implementation", + &["state"] + ) + .unwrap(); + + let started_gcs = gcs.get_metric_with_label_values(&["pending"]).unwrap(); + let completed_gcs = gcs.get_metric_with_label_values(&["completed"]).unwrap(); + + let failed_gcs = metrics::register_int_counter_vec!( + "pageserver_layer_failed_gcs_count", + "Different reasons for garbage collections to have failed", + &["reason"] + ) + .unwrap(); + + let rare_counters = metrics::register_int_counter_vec!( + "pageserver_layer_assumed_rare_count", + "Times unexpected or assumed rare event happened", + &["event"] + ) + .unwrap(); + + Self { + started_evictions, + completed_evictions, + cancelled_evictions, + + started_gcs, + completed_gcs, + failed_gcs, + + rare_counters, + } + } +} + +impl LayerImplMetrics { + fn inc_started_evictions(&self) { + self.started_evictions.inc(); + } + fn inc_completed_evictions(&self) { + self.completed_evictions.inc(); + } + fn inc_eviction_cancelled(&self, reason: EvictionCancelled) { + self.cancelled_evictions + .get_metric_with_label_values(&[reason.as_str()]) + .unwrap() + .inc() + } + + fn inc_started_gcs(&self) { + self.started_gcs.inc(); + } + fn inc_completed_gcs(&self) { + self.completed_gcs.inc(); + } + fn inc_gcs_failed(&self, reason: GcFailed) { + self.failed_gcs + .get_metric_with_label_values(&[reason.as_str()]) + .unwrap() + .inc(); + } + + /// Counted separatedly from failed gcs because we will complete the gc attempt regardless of + /// failure to delete local file. + fn inc_gc_removes_failed(&self) { + self.rare_counters + .get_metric_with_label_values(&["gc_remove_failed"]) + .unwrap() + .inc(); + } + + /// Expected rare because requires a race with `evict_blocking` and + /// `get_or_maybe_download`. + fn inc_retried_get_or_maybe_download(&self) { + self.rare_counters + .get_metric_with_label_values(&["retried_gomd"]) + .unwrap() + .inc(); + } + + /// Expected rare because cancellations are unexpected + fn inc_download_completed_without_requester(&self) { + self.rare_counters + .get_metric_with_label_values(&["download_completed_without"]) + .unwrap() + .inc(); + } + + /// Expected rare because cancellations are unexpected + fn inc_download_failed_without_requester(&self) { + self.rare_counters + .get_metric_with_label_values(&["download_failed_without"]) + .unwrap() + .inc(); + } + + /// The Weak in ResidentOrWantedEvicted::WantedEvicted was successfully upgraded. + /// + /// If this counter is always zero, we should replace ResidentOrWantedEvicted type with an + /// Option. + fn inc_raced_wanted_evicted_accesses(&self) { + self.rare_counters + .get_metric_with_label_values(&["raced_wanted_evicted"]) + .unwrap() + .inc(); + } + + /// These are only expected for [`Self::inc_download_completed_without_requester`] amount when + /// running with remote storage. + fn inc_init_needed_no_download(&self) { + self.rare_counters + .get_metric_with_label_values(&["init_needed_no_download"]) + .unwrap() + .inc(); + } + + /// Expected rare because all layer files should be readable and good + fn inc_permanent_loading_failures(&self) { + self.rare_counters + .get_metric_with_label_values(&["permanent_loading_failure"]) + .unwrap() + .inc(); + } +} + +enum EvictionCancelled { + LayerGone, + TimelineGone, + VersionCheckFailed, + FileNotFound, + RemoveFailed, +} + +impl EvictionCancelled { + fn as_str(&self) -> &'static str { + match self { + EvictionCancelled::LayerGone => "layer_gone", + EvictionCancelled::TimelineGone => "timeline_gone", + EvictionCancelled::VersionCheckFailed => "version_check_fail", + EvictionCancelled::FileNotFound => "file_not_found", + EvictionCancelled::RemoveFailed => "remove_failed", + } + } +} + +enum GcFailed { + TimelineGone, + DeleteSchedulingFailed, +} + +impl GcFailed { + fn as_str(&self) -> &'static str { + match self { + GcFailed::TimelineGone => "timeline_gone", + GcFailed::DeleteSchedulingFailed => "delete_scheduling_failed", + } + } +} + +static LAYER_IMPL_METRICS: once_cell::sync::Lazy = + once_cell::sync::Lazy::new(LayerImplMetrics::default); diff --git a/pageserver/src/tenant/storage_layer/layer_desc.rs b/pageserver/src/tenant/storage_layer/layer_desc.rs index 6d6ab801d8..2e0b0b3e64 100644 --- a/pageserver/src/tenant/storage_layer/layer_desc.rs +++ b/pageserver/src/tenant/storage_layer/layer_desc.rs @@ -1,4 +1,3 @@ -use anyhow::Result; use core::fmt::Display; use std::ops::Range; use utils::{ @@ -6,7 +5,7 @@ use utils::{ lsn::Lsn, }; -use crate::{context::RequestContext, repository::Key}; +use crate::repository::Key; use super::{DeltaFileName, ImageFileName, LayerFileName}; @@ -100,6 +99,22 @@ impl PersistentLayerDesc { } } + pub fn from_filename( + tenant_id: TenantId, + timeline_id: TimelineId, + filename: LayerFileName, + file_size: u64, + ) -> Self { + match filename { + LayerFileName::Image(i) => { + Self::new_img(tenant_id, timeline_id, i.key_range, i.lsn, file_size) + } + LayerFileName::Delta(d) => { + Self::new_delta(tenant_id, timeline_id, d.key_range, d.lsn_range, file_size) + } + } + } + /// Get the LSN that the image layer covers. pub fn image_layer_lsn(&self) -> Lsn { assert!(!self.is_delta); @@ -173,21 +188,31 @@ impl PersistentLayerDesc { self.is_delta } - pub fn dump(&self, _verbose: bool, _ctx: &RequestContext) -> Result<()> { - println!( - "----- layer for ten {} tli {} keys {}-{} lsn {}-{} is_delta {} is_incremental {} size {} ----", - self.tenant_id, - self.timeline_id, - self.key_range.start, - self.key_range.end, - self.lsn_range.start, - self.lsn_range.end, - self.is_delta, - self.is_incremental(), - self.file_size, - ); - - Ok(()) + pub fn dump(&self) { + if self.is_delta { + println!( + "----- delta layer for ten {} tli {} keys {}-{} lsn {}-{} is_incremental {} size {} ----", + self.tenant_id, + self.timeline_id, + self.key_range.start, + self.key_range.end, + self.lsn_range.start, + self.lsn_range.end, + self.is_incremental(), + self.file_size, + ); + } else { + println!( + "----- image layer for ten {} tli {} key {}-{} at {} is_incremental {} size {} ----", + self.tenant_id, + self.timeline_id, + self.key_range.start, + self.key_range.end, + self.image_layer_lsn(), + self.is_incremental(), + self.file_size + ); + } } pub fn file_size(&self) -> u64 { diff --git a/pageserver/src/tenant/storage_layer/remote_layer.rs b/pageserver/src/tenant/storage_layer/remote_layer.rs deleted file mode 100644 index 94bebb5f6e..0000000000 --- a/pageserver/src/tenant/storage_layer/remote_layer.rs +++ /dev/null @@ -1,216 +0,0 @@ -//! A RemoteLayer is an in-memory placeholder for a layer file that exists -//! in remote storage. -//! -use crate::config::PageServerConf; -use crate::context::RequestContext; -use crate::repository::Key; -use crate::tenant::remote_timeline_client::index::LayerFileMetadata; -use crate::tenant::storage_layer::{Layer, ValueReconstructResult, ValueReconstructState}; -use crate::tenant::timeline::layer_manager::LayerManager; -use anyhow::{bail, Result}; -use camino::Utf8PathBuf; -use pageserver_api::models::HistoricLayerInfo; -use std::ops::Range; -use std::sync::Arc; - -use utils::{ - id::{TenantId, TimelineId}, - lsn::Lsn, -}; - -use super::filename::{DeltaFileName, ImageFileName}; -use super::{ - AsLayerDesc, DeltaLayer, ImageLayer, LayerAccessStats, LayerAccessStatsReset, - LayerResidenceStatus, PersistentLayer, PersistentLayerDesc, -}; - -/// RemoteLayer is a not yet downloaded [`ImageLayer`] or -/// [`DeltaLayer`]. -/// -/// RemoteLayer might be downloaded on-demand during operations which are -/// allowed download remote layers and during which, it gets replaced with a -/// concrete `DeltaLayer` or `ImageLayer`. -/// -/// See: [`crate::context::RequestContext`] for authorization to download -pub struct RemoteLayer { - pub desc: PersistentLayerDesc, - - pub layer_metadata: LayerFileMetadata, - - access_stats: LayerAccessStats, - - pub(crate) ongoing_download: Arc, - - /// Has `LayerMap::replace` failed for this (true) or not (false). - /// - /// Used together with [`ongoing_download`] semaphore in `Timeline::download_remote_layer`. - /// The field is used to mark a RemoteLayer permanently (until restart or ignore+load) - /// unprocessable, because a LayerMap::replace failed. - /// - /// It is very unlikely to accumulate these in the Timeline's LayerMap, but having this avoids - /// a possible fast loop between `Timeline::get_reconstruct_data` and - /// `Timeline::download_remote_layer`, which also logs. - /// - /// [`ongoing_download`]: Self::ongoing_download - pub(crate) download_replacement_failure: std::sync::atomic::AtomicBool, -} - -impl std::fmt::Debug for RemoteLayer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("RemoteLayer") - .field("file_name", &self.desc.filename()) - .field("layer_metadata", &self.layer_metadata) - .field("is_incremental", &self.desc.is_incremental()) - .finish() - } -} - -#[async_trait::async_trait] -impl Layer for RemoteLayer { - async fn get_value_reconstruct_data( - &self, - _key: Key, - _lsn_range: Range, - _reconstruct_state: &mut ValueReconstructState, - _ctx: &RequestContext, - ) -> Result { - Err(anyhow::anyhow!("layer {self} needs to be downloaded")) - } -} - -/// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. -impl std::fmt::Display for RemoteLayer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.layer_desc().short_id()) - } -} - -impl AsLayerDesc for RemoteLayer { - fn layer_desc(&self) -> &PersistentLayerDesc { - &self.desc - } -} - -impl PersistentLayer for RemoteLayer { - fn local_path(&self) -> Option { - None - } - - fn delete_resident_layer_file(&self) -> Result<()> { - bail!("remote layer has no layer file"); - } - - fn downcast_remote_layer<'a>(self: Arc) -> Option> { - Some(self) - } - - fn is_remote_layer(&self) -> bool { - true - } - - fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { - let layer_file_name = self.layer_desc().filename().file_name(); - let lsn_range = self.layer_desc().lsn_range.clone(); - - if self.desc.is_delta { - HistoricLayerInfo::Delta { - layer_file_name, - layer_file_size: self.layer_metadata.file_size(), - lsn_start: lsn_range.start, - lsn_end: lsn_range.end, - remote: true, - access_stats: self.access_stats.as_api_model(reset), - } - } else { - HistoricLayerInfo::Image { - layer_file_name, - layer_file_size: self.layer_metadata.file_size(), - lsn_start: lsn_range.start, - remote: true, - access_stats: self.access_stats.as_api_model(reset), - } - } - } - - fn access_stats(&self) -> &LayerAccessStats { - &self.access_stats - } -} - -impl RemoteLayer { - pub fn new_img( - tenantid: TenantId, - timelineid: TimelineId, - fname: &ImageFileName, - layer_metadata: &LayerFileMetadata, - access_stats: LayerAccessStats, - ) -> RemoteLayer { - RemoteLayer { - desc: PersistentLayerDesc::new_img( - tenantid, - timelineid, - fname.key_range.clone(), - fname.lsn, - layer_metadata.file_size(), - ), - layer_metadata: layer_metadata.clone(), - ongoing_download: Arc::new(tokio::sync::Semaphore::new(1)), - download_replacement_failure: std::sync::atomic::AtomicBool::default(), - access_stats, - } - } - - pub fn new_delta( - tenantid: TenantId, - timelineid: TimelineId, - fname: &DeltaFileName, - layer_metadata: &LayerFileMetadata, - access_stats: LayerAccessStats, - ) -> RemoteLayer { - RemoteLayer { - desc: PersistentLayerDesc::new_delta( - tenantid, - timelineid, - fname.key_range.clone(), - fname.lsn_range.clone(), - layer_metadata.file_size(), - ), - layer_metadata: layer_metadata.clone(), - ongoing_download: Arc::new(tokio::sync::Semaphore::new(1)), - download_replacement_failure: std::sync::atomic::AtomicBool::default(), - access_stats, - } - } - - /// Create a Layer struct representing this layer, after it has been downloaded. - pub(crate) fn create_downloaded_layer( - &self, - _layer_map_lock_held_witness: &LayerManager, - conf: &'static PageServerConf, - file_size: u64, - ) -> Arc { - if self.desc.is_delta { - let fname = self.desc.delta_file_name(); - Arc::new(DeltaLayer::new( - conf, - self.desc.timeline_id, - self.desc.tenant_id, - &fname, - file_size, - self.access_stats - .clone_for_residence_change(LayerResidenceStatus::Resident), - )) - } else { - let fname = self.desc.image_file_name(); - Arc::new(ImageLayer::new( - conf, - self.desc.timeline_id, - self.desc.tenant_id, - &fname, - file_size, - self.access_stats - .clone_for_residence_change(LayerResidenceStatus::Resident), - )) - } - } -} diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 91aad10210..278290eef4 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -11,18 +11,16 @@ use anyhow::{anyhow, bail, ensure, Context, Result}; use bytes::Bytes; use camino::{Utf8Path, Utf8PathBuf}; use fail::fail_point; -use futures::StreamExt; use itertools::Itertools; use pageserver_api::models::{ - DownloadRemoteLayersTaskInfo, DownloadRemoteLayersTaskSpawnRequest, - DownloadRemoteLayersTaskState, LayerMapInfo, LayerResidenceEventReason, LayerResidenceStatus, - TimelineState, + DownloadRemoteLayersTaskInfo, DownloadRemoteLayersTaskSpawnRequest, LayerMapInfo, TimelineState, }; -use remote_storage::GenericRemoteStorage; use serde_with::serde_as; use storage_broker::BrokerClientChannel; -use tokio::runtime::Handle; -use tokio::sync::{oneshot, watch, TryAcquireError}; +use tokio::{ + runtime::Handle, + sync::{oneshot, watch, TryAcquireError}, +}; use tokio_util::sync::CancellationToken; use tracing::*; use utils::id::TenantTimelineId; @@ -39,10 +37,11 @@ use crate::context::{ AccessStatsBehavior, DownloadBehavior, RequestContext, RequestContextBuilder, }; use crate::deletion_queue::DeletionQueueClient; -use crate::tenant::remote_timeline_client::index::LayerFileMetadata; use crate::tenant::storage_layer::delta_layer::DeltaEntry; use crate::tenant::storage_layer::{ - DeltaLayerWriter, ImageLayerWriter, InMemoryLayer, LayerAccessStats, LayerFileName, RemoteLayer, + AsLayerDesc, DeltaLayerWriter, EvictionError, ImageLayerWriter, InMemoryLayer, Layer, + LayerAccessStatsReset, LayerFileName, ResidentLayer, ValueReconstructResult, + ValueReconstructState, }; use crate::tenant::tasks::{BackgroundLoopKind, RateLimitError}; use crate::tenant::timeline::logical_size::CurrentLogicalSize; @@ -50,14 +49,12 @@ use crate::tenant::{ layer_map::{LayerMap, SearchResult}, metadata::{save_metadata, TimelineMetadata}, par_fsync, - storage_layer::{PersistentLayer, ValueReconstructResult, ValueReconstructState}, }; use crate::config::PageServerConf; use crate::keyspace::{KeyPartitioning, KeySpace, KeySpaceRandomAccum}; use crate::metrics::{ TimelineMetrics, MATERIALIZED_PAGE_CACHE_HIT, MATERIALIZED_PAGE_CACHE_HIT_DIRECT, - UNEXPECTED_ONDEMAND_DOWNLOADS, }; use crate::pgdatadir_mapping::LsnForTimestamp; use crate::pgdatadir_mapping::{is_rel_fsm_block_key, is_rel_vm_block_key}; @@ -93,9 +90,6 @@ use self::walreceiver::{WalReceiver, WalReceiverConf}; use super::config::TenantConf; use super::remote_timeline_client::index::IndexPart; use super::remote_timeline_client::RemoteTimelineClient; -use super::storage_layer::{ - AsLayerDesc, DeltaLayer, ImageLayer, LayerAccessStatsReset, PersistentLayerDesc, -}; use super::{debug_assert_current_span_has_tenant_and_timeline_id, AttachedTenantConf}; #[derive(Debug, PartialEq, Eq, Clone, Copy)] @@ -161,7 +155,7 @@ pub struct Timeline { /// /// This duplicates the generation stored in LocationConf, but that structure is mutable: /// this copy enforces the invariant that generatio doesn't change during a Tenant's lifetime. - generation: Generation, + pub(crate) generation: Generation, pub pg_version: u32, @@ -677,13 +671,11 @@ impl Timeline { } /// Outermost timeline compaction operation; downloads needed layers. - pub async fn compact( + pub(crate) async fn compact( self: &Arc, cancel: &CancellationToken, ctx: &RequestContext, - ) -> anyhow::Result<()> { - const ROUNDS: usize = 2; - + ) -> Result<(), CompactionError> { // this wait probably never needs any "long time spent" logging, because we already nag if // compaction task goes over it's period (20s) which is quite often in production. let _permit = match super::tasks::concurrent_background_tasks_rate_limit( @@ -705,87 +697,6 @@ impl Timeline { return Ok(()); } - // retry two times to allow first round to find layers which need to be downloaded, then - // download them, then retry compaction - for round in 0..ROUNDS { - // should we error out with the most specific error? - let last_round = round == ROUNDS - 1; - - let res = self.compact_inner(ctx).await; - - // If `create_image_layers' or `compact_level0` scheduled any - // uploads or deletions, but didn't update the index file yet, - // do it now. - // - // This isn't necessary for correctness, the remote state is - // consistent without the uploads and deletions, and we would - // update the index file on next flush iteration too. But it - // could take a while until that happens. - // - // Additionally, only do this once before we return from this function. - if last_round || res.is_ok() { - if let Some(remote_client) = &self.remote_client { - remote_client.schedule_index_upload_for_file_changes()?; - } - } - - let rls = match res { - Ok(()) => return Ok(()), - Err(CompactionError::DownloadRequired(rls)) if !last_round => { - // this can be done at most one time before exiting, waiting - rls - } - Err(CompactionError::DownloadRequired(rls)) => { - anyhow::bail!("Compaction requires downloading multiple times (last was {} layers), possibly battling against eviction", rls.len()) - } - Err(CompactionError::ShuttingDown) => { - return Ok(()); - } - Err(CompactionError::Other(e)) => { - return Err(e); - } - }; - - // this path can be visited in the second round of retrying, if first one found that we - // must first download some remote layers - let total = rls.len(); - - let mut downloads = rls - .into_iter() - .map(|rl| self.download_remote_layer(rl)) - .collect::>(); - - let mut failed = 0; - - loop { - tokio::select! { - _ = cancel.cancelled() => anyhow::bail!("Cancelled while downloading remote layers"), - res = downloads.next() => { - match res { - Some(Ok(())) => {}, - Some(Err(e)) => { - warn!("Downloading remote layer for compaction failed: {e:#}"); - failed += 1; - } - None => break, - } - } - } - } - - if failed != 0 { - anyhow::bail!("{failed} out of {total} layers failed to download, retrying later"); - } - - // if everything downloaded fine, lets try again - } - - unreachable!("retry loop exits") - } - - /// Compaction which might need to be retried after downloading remote layers. - async fn compact_inner(self: &Arc, ctx: &RequestContext) -> Result<(), CompactionError> { - // // High level strategy for compaction / image creation: // // 1. First, calculate the desired "partitioning" of the @@ -830,6 +741,7 @@ impl Timeline { // Define partitioning schema if needed + // FIXME: the match should only cover repartitioning, not the next steps match self .repartition( self.get_last_record_lsn(), @@ -846,13 +758,13 @@ impl Timeline { // 2. Create new image layers for partitions that have been modified // "enough". - let layer_paths_to_upload = self + let layers = self .create_image_layers(&partitioning, lsn, false, &image_ctx) .await .map_err(anyhow::Error::from)?; if let Some(remote_client) = &self.remote_client { - for (path, layer_metadata) in layer_paths_to_upload { - remote_client.schedule_layer_file_upload(&path, &layer_metadata)?; + for layer in layers { + remote_client.schedule_layer_file_upload(layer)?; } } @@ -861,6 +773,13 @@ impl Timeline { self.compact_level0(layer_removal_cs.clone(), target_file_size, ctx) .await?; timer.stop_and_record(); + + if let Some(remote_client) = &self.remote_client { + // should any new image layer been created, not uploading index_part will + // result in a mismatch between remote_physical_size and layermap calculated + // size, which will fail some tests, but should not be an issue otherwise. + remote_client.schedule_index_upload_for_file_changes()?; + } } Err(err) => { // no partitioning? This is normal, if the timeline was just created @@ -1116,14 +1035,13 @@ impl Timeline { let Some(layer) = self.find_layer(layer_file_name).await else { return Ok(None); }; - let Some(remote_layer) = layer.downcast_remote_layer() else { - return Ok(Some(false)); - }; + if self.remote_client.is_none() { return Ok(Some(false)); } - self.download_remote_layer(remote_layer).await?; + layer.download().await?; + Ok(Some(true)) } @@ -1133,6 +1051,13 @@ impl Timeline { let Some(local_layer) = self.find_layer(layer_file_name).await else { return Ok(None); }; + + let Some(local_layer) = local_layer.keep_resident().await? else { + return Ok(Some(false)); + }; + + let local_layer: Layer = local_layer.into(); + let remote_client = self .remote_client .as_ref() @@ -1140,7 +1065,7 @@ impl Timeline { let cancel = CancellationToken::new(); let results = self - .evict_layer_batch(remote_client, &[local_layer], cancel) + .evict_layer_batch(remote_client, &[local_layer], &cancel) .await?; assert_eq!(results.len(), 1); let result: Option> = results.into_iter().next().unwrap(); @@ -1152,31 +1077,22 @@ impl Timeline { } /// Evict a batch of layers. - /// - /// GenericRemoteStorage reference is required as a (witness)[witness_article] for "remote storage is configured." - /// - /// [witness_article]: https://willcrichton.net/rust-api-type-patterns/witnesses.html pub(crate) async fn evict_layers( &self, - _: &GenericRemoteStorage, - layers_to_evict: &[Arc], - cancel: CancellationToken, + layers_to_evict: &[Layer], + cancel: &CancellationToken, ) -> anyhow::Result>>> { - let remote_client = self.remote_client.clone().expect( - "GenericRemoteStorage is configured, so timeline must have RemoteTimelineClient", - ); + let remote_client = self + .remote_client + .as_ref() + .context("timeline must have RemoteTimelineClient")?; - self.evict_layer_batch(&remote_client, layers_to_evict, cancel) + self.evict_layer_batch(remote_client, layers_to_evict, cancel) .await } /// Evict multiple layers at once, continuing through errors. /// - /// Try to evict the given `layers_to_evict` by - /// - /// 1. Replacing the given layer object in the layer map with a corresponding [`RemoteLayer`] object. - /// 2. Deleting the now unreferenced layer file from disk. - /// /// The `remote_client` should be this timeline's `self.remote_client`. /// We make the caller provide it so that they are responsible for handling the case /// where someone wants to evict the layer but no remote storage is configured. @@ -1185,17 +1101,15 @@ impl Timeline { /// If `Err()` is returned, no eviction was attempted. /// Each position of `Ok(results)` corresponds to the layer in `layers_to_evict`. /// Meaning of each `result[i]`: - /// - `Some(Err(...))` if layer replacement failed for an unexpected reason - /// - `Some(Ok(true))` if everything went well. - /// - `Some(Ok(false))` if there was an expected reason why the layer could not be replaced, e.g.: - /// - evictee was not yet downloaded + /// - `Some(Err(...))` if layer replacement failed for some reason /// - replacement failed for an expectable reason (e.g., layer removed by GC before we grabbed all locks) + /// - `Some(Ok(()))` if everything went well. /// - `None` if no eviction attempt was made for the layer because `cancel.is_cancelled() == true`. async fn evict_layer_batch( &self, remote_client: &Arc, - layers_to_evict: &[Arc], - cancel: CancellationToken, + layers_to_evict: &[Layer], + cancel: &CancellationToken, ) -> anyhow::Result>>> { // ensure that the layers have finished uploading // (don't hold the layer_removal_cs while we do it, we're not removing anything yet) @@ -1205,7 +1119,7 @@ impl Timeline { .context("wait for layer upload ops to complete")?; // now lock out layer removal (compaction, gc, timeline deletion) - let layer_removal_guard = self.layer_removal_cs.lock().await; + let _layer_removal_guard = self.layer_removal_cs.lock().await; { // to avoid racing with detach and delete_timeline @@ -1216,163 +1130,40 @@ impl Timeline { ); } - // start the batch update - let mut guard = self.layers.write().await; let mut results = Vec::with_capacity(layers_to_evict.len()); - - for l in layers_to_evict.iter() { - let res = if cancel.is_cancelled() { - None - } else { - Some(self.evict_layer_batch_impl(&layer_removal_guard, l, &mut guard)) - }; - results.push(res); + for _ in 0..layers_to_evict.len() { + results.push(None); } - // commit the updates & release locks - drop_wlock(guard); - drop(layer_removal_guard); + let mut js = tokio::task::JoinSet::new(); + + for (i, l) in layers_to_evict.iter().enumerate() { + js.spawn({ + let l = l.to_owned(); + let remote_client = remote_client.clone(); + async move { (i, l.evict_and_wait(&remote_client).await) } + }); + } + + let join = async { + while let Some(next) = js.join_next().await { + match next { + Ok((i, res)) => results[i] = Some(res), + Err(je) if je.is_cancelled() => unreachable!("not used"), + Err(je) if je.is_panic() => { /* already logged */ } + Err(je) => tracing::error!("unknown JoinError: {je:?}"), + } + } + }; + + tokio::select! { + _ = cancel.cancelled() => {}, + _ = join => {} + } assert_eq!(results.len(), layers_to_evict.len()); Ok(results) } - - fn evict_layer_batch_impl( - &self, - _layer_removal_cs: &tokio::sync::MutexGuard<'_, ()>, - local_layer: &Arc, - layer_mgr: &mut LayerManager, - ) -> Result<(), EvictionError> { - if local_layer.is_remote_layer() { - return Err(EvictionError::CannotEvictRemoteLayer); - } - - let layer_file_size = local_layer.layer_desc().file_size; - - let local_layer_mtime = local_layer - .local_path() - .expect("local layer should have a local path") - .metadata() - // when the eviction fails because we have already deleted the layer in compaction for - // example, a NotFound error bubbles up from here. - .map_err(|e| { - if e.kind() == std::io::ErrorKind::NotFound { - EvictionError::FileNotFound - } else { - EvictionError::StatFailed(e) - } - })? - .modified() - .map_err(EvictionError::StatFailed)?; - - let local_layer_residence_duration = - match SystemTime::now().duration_since(local_layer_mtime) { - Err(e) => { - warn!(layer = %local_layer, "layer mtime is in the future: {}", e); - None - } - Ok(delta) => Some(delta), - }; - - // RemoteTimelineClient holds the metadata on layers' remote generations, so - // query it to construct a RemoteLayer. - let layer_metadata = self - .remote_client - .as_ref() - .expect("Eviction is not called without remote storage") - .get_layer_metadata(&local_layer.filename()) - .map_err(EvictionError::LayerNotFound)? - .ok_or_else(|| { - EvictionError::LayerNotFound(anyhow::anyhow!("Layer not in remote metadata")) - })?; - if layer_metadata.file_size() != layer_file_size { - return Err(EvictionError::MetadataInconsistency(format!( - "Layer size {layer_file_size} doesn't match remote metadata file size {}", - layer_metadata.file_size() - ))); - } - - let new_remote_layer = Arc::new(match local_layer.filename() { - LayerFileName::Image(image_name) => RemoteLayer::new_img( - self.tenant_id, - self.timeline_id, - &image_name, - &layer_metadata, - local_layer - .access_stats() - .clone_for_residence_change(LayerResidenceStatus::Evicted), - ), - LayerFileName::Delta(delta_name) => RemoteLayer::new_delta( - self.tenant_id, - self.timeline_id, - &delta_name, - &layer_metadata, - local_layer - .access_stats() - .clone_for_residence_change(LayerResidenceStatus::Evicted), - ), - }); - - assert_eq!(local_layer.layer_desc(), new_remote_layer.layer_desc()); - - layer_mgr - .replace_and_verify(local_layer.clone(), new_remote_layer) - .map_err(EvictionError::LayerNotFound)?; - - if let Err(e) = local_layer.delete_resident_layer_file() { - // this should never happen, because of layer_removal_cs usage and above stat - // access for mtime - error!("failed to remove layer file on evict after replacement: {e:#?}"); - } - // Always decrement the physical size gauge, even if we failed to delete the file. - // Rationale: we already replaced the layer with a remote layer in the layer map, - // and any subsequent download_remote_layer will - // 1. overwrite the file on disk and - // 2. add the downloaded size to the resident size gauge. - // - // If there is no re-download, and we restart the pageserver, then load_layer_map - // will treat the file as a local layer again, count it towards resident size, - // and it'll be like the layer removal never happened. - // The bump in resident size is perhaps unexpected but overall a robust behavior. - self.metrics.resident_physical_size_sub(layer_file_size); - self.metrics.evictions.inc(); - - if let Some(delta) = local_layer_residence_duration { - self.metrics - .evictions_with_low_residence_duration - .read() - .unwrap() - .observe(delta); - info!(layer=%local_layer, residence_millis=delta.as_millis(), "evicted layer after known residence period"); - } else { - info!(layer=%local_layer, "evicted layer after unknown residence period"); - } - - Ok(()) - } -} - -#[derive(Debug, thiserror::Error)] -pub(crate) enum EvictionError { - #[error("cannot evict a remote layer")] - CannotEvictRemoteLayer, - /// Most likely the to-be evicted layer has been deleted by compaction or gc which use the same - /// locks, so they got to execute before the eviction. - #[error("file backing the layer has been removed already")] - FileNotFound, - #[error("stat failed")] - StatFailed(#[source] std::io::Error), - /// In practice, this can be a number of things, but lets assume it means only this. - /// - /// This case includes situations such as the Layer was evicted and redownloaded in between, - /// because the file existed before an replacement attempt was made but now the Layers are - /// different objects in memory. - #[error("layer was no longer part of LayerMap")] - LayerNotFound(#[source] anyhow::Error), - - /// This should never happen - #[error("Metadata inconsistency")] - MetadataInconsistency(String), } /// Number of times we will compute partition within a checkpoint distance. @@ -1709,11 +1500,12 @@ impl Timeline { // Scan timeline directory and create ImageFileName and DeltaFilename // structs representing all files on disk let timeline_path = self.conf.timeline_path(&self.tenant_id, &self.timeline_id); - let (conf, tenant_id, timeline_id) = (self.conf, self.tenant_id, self.timeline_id); + let conf = self.conf; let span = tracing::Span::current(); // Copy to move into the task we're about to spawn let generation = self.generation; + let this = self.myself.upgrade().expect("&self method holds the arc"); let (loaded_layers, needs_cleanup, total_physical_size) = tokio::task::spawn_blocking({ move || { @@ -1807,44 +1599,16 @@ impl Timeline { Image(i) => assert!(i.lsn <= disk_consistent_lsn), } - let status = match &decision { - UseLocal(_) => LayerResidenceStatus::Resident, - Evicted(_) | UseRemote { .. } => LayerResidenceStatus::Evicted, - }; + tracing::debug!(layer=%name, ?decision, "applied"); - tracing::debug!(layer=%name, ?decision, ?status, "applied"); - - let stats = LayerAccessStats::for_loading_layer(status); - - let layer: Arc = match (name, &decision) { - (Delta(d), UseLocal(m)) => { + let layer = match decision { + UseLocal(m) => { total_physical_size += m.file_size(); - Arc::new(DeltaLayer::new( - conf, - timeline_id, - tenant_id, - &d, - m.file_size(), - stats, - )) + Layer::for_resident(conf, &this, name, m).drop_eviction_guard() } - (Image(i), UseLocal(m)) => { - total_physical_size += m.file_size(); - Arc::new(ImageLayer::new( - conf, - timeline_id, - tenant_id, - &i, - m.file_size(), - stats, - )) + Evicted(remote) | UseRemote { remote, .. } => { + Layer::for_evicted(conf, &this, name, remote) } - (Delta(d), Evicted(remote) | UseRemote { remote, .. }) => Arc::new( - RemoteLayer::new_delta(tenant_id, timeline_id, &d, remote, stats), - ), - (Image(i), Evicted(remote) | UseRemote { remote, .. }) => Arc::new( - RemoteLayer::new_img(tenant_id, timeline_id, &i, remote, stats), - ), }; loaded_layers.push(layer); @@ -1861,7 +1625,7 @@ impl Timeline { guard.initialize_local_layers(loaded_layers, disk_consistent_lsn + 1); if let Some(rtc) = self.remote_client.as_ref() { - rtc.schedule_layer_file_deletion(needs_cleanup)?; + rtc.schedule_layer_file_deletion(&needs_cleanup)?; rtc.schedule_index_upload_for_file_changes()?; // Tenant::create_timeline will wait for these uploads to happen before returning, or // on retry. @@ -1871,7 +1635,6 @@ impl Timeline { "loaded layer map with {} layers at {}, total physical size: {}", num_layers, disk_consistent_lsn, total_physical_size ); - self.metrics.resident_physical_size_add(total_physical_size); timer.stop_and_record(); Ok(()) @@ -2195,7 +1958,7 @@ impl Timeline { } } - async fn find_layer(&self, layer_file_name: &str) -> Option> { + async fn find_layer(&self, layer_file_name: &str) -> Option { let guard = self.layers.read().await; for historic_layer in guard.layer_map().iter_historic_layers() { let historic_layer_name = historic_layer.filename().file_name(); @@ -2214,20 +1977,9 @@ trait TraversalLayerExt { fn traversal_id(&self) -> TraversalId; } -impl TraversalLayerExt for Arc { +impl TraversalLayerExt for Layer { fn traversal_id(&self) -> TraversalId { - let timeline_id = self.layer_desc().timeline_id; - match self.local_path() { - Some(local_path) => { - debug_assert!(local_path.to_string().contains(&format!("{}", timeline_id)), - "need timeline ID to uniquely identify the layer when traversal crosses ancestor boundary", - ); - format!("{local_path}") - } - None => { - format!("remote {}/{self}", timeline_id) - } - } + self.local_path().to_string() } } @@ -2386,174 +2138,107 @@ impl Timeline { continue 'outer; } - #[allow(clippy::never_loop)] // see comment at bottom of this loop - 'layer_map_search: loop { - let remote_layer = { - let guard = timeline.layers.read().await; - let layers = guard.layer_map(); + let guard = timeline.layers.read().await; + let layers = guard.layer_map(); - // Check the open and frozen in-memory layers first, in order from newest - // to oldest. - if let Some(open_layer) = &layers.open_layer { - let start_lsn = open_layer.get_lsn_range().start; - if cont_lsn > start_lsn { - //info!("CHECKING for {} at {} on open layer {}", key, cont_lsn, open_layer.filename().display()); - // Get all the data needed to reconstruct the page version from this layer. - // But if we have an older cached page image, no need to go past that. - let lsn_floor = max(cached_lsn + 1, start_lsn); - result = match open_layer - .get_value_reconstruct_data( - key, - lsn_floor..cont_lsn, - reconstruct_state, - ctx, - ) - .await - { - Ok(result) => result, - Err(e) => return Err(PageReconstructError::from(e)), - }; - cont_lsn = lsn_floor; - // metrics: open_layer does not count as fs access, so we are not updating `read_count` - traversal_path.push(( - result, - cont_lsn, - Box::new({ - let open_layer = Arc::clone(open_layer); - move || open_layer.traversal_id() - }), - )); - continue 'outer; - } - } - for frozen_layer in layers.frozen_layers.iter().rev() { - let start_lsn = frozen_layer.get_lsn_range().start; - if cont_lsn > start_lsn { - //info!("CHECKING for {} at {} on frozen layer {}", key, cont_lsn, frozen_layer.filename().display()); - let lsn_floor = max(cached_lsn + 1, start_lsn); - result = match frozen_layer - .get_value_reconstruct_data( - key, - lsn_floor..cont_lsn, - reconstruct_state, - ctx, - ) - .await - { - Ok(result) => result, - Err(e) => return Err(PageReconstructError::from(e)), - }; - cont_lsn = lsn_floor; - // metrics: open_layer does not count as fs access, so we are not updating `read_count` - traversal_path.push(( - result, - cont_lsn, - Box::new({ - let frozen_layer = Arc::clone(frozen_layer); - move || frozen_layer.traversal_id() - }), - )); - continue 'outer; - } - } - - if let Some(SearchResult { lsn_floor, layer }) = layers.search(key, cont_lsn) { - let layer = guard.get_from_desc(&layer); - // If it's a remote layer, download it and retry. - if let Some(remote_layer) = - super::storage_layer::downcast_remote_layer(&layer) - { - // TODO: push a breadcrumb to 'traversal_path' to record the fact that - // we downloaded / would need to download this layer. - remote_layer // download happens outside the scope of `layers` guard object - } else { - // Get all the data needed to reconstruct the page version from this layer. - // But if we have an older cached page image, no need to go past that. - let lsn_floor = max(cached_lsn + 1, lsn_floor); - result = match layer - .get_value_reconstruct_data( - key, - lsn_floor..cont_lsn, - reconstruct_state, - ctx, - ) - .await - { - Ok(result) => result, - Err(e) => return Err(PageReconstructError::from(e)), - }; - cont_lsn = lsn_floor; - *read_count += 1; - traversal_path.push(( - result, - cont_lsn, - Box::new({ - let layer = Arc::clone(&layer); - move || layer.traversal_id() - }), - )); - continue 'outer; - } - } else if timeline.ancestor_timeline.is_some() { - // Nothing on this timeline. Traverse to parent - result = ValueReconstructResult::Continue; - cont_lsn = Lsn(timeline.ancestor_lsn.0 + 1); - continue 'outer; - } else { - // Nothing found - result = ValueReconstructResult::Missing; - continue 'outer; - } - }; - // Download the remote_layer and replace it in the layer map. - // For that, we need to release the mutex. Otherwise, we'd deadlock. - // - // The control flow is so weird here because `drop(layers)` inside - // the if stmt above is not enough for current rustc: it requires - // that the layers lock guard is not in scope across the download - // await point. - let remote_layer_as_persistent: Arc = - Arc::clone(&remote_layer) as Arc; - let id = remote_layer_as_persistent.traversal_id(); - info!( - "need remote layer {} for task kind {:?}", - id, - ctx.task_kind() - ); - - // The next layer doesn't exist locally. Need to download it. - // (The control flow is a bit complicated here because we must drop the 'layers' - // lock before awaiting on the Future.) - match ( - ctx.download_behavior(), - self.conf.ondemand_download_behavior_treat_error_as_warn, - ) { - (DownloadBehavior::Download, _) => { - info!( - "on-demand downloading remote layer {id} for task kind {:?}", - ctx.task_kind() - ); - timeline.download_remote_layer(remote_layer).await?; - continue 'layer_map_search; - } - (DownloadBehavior::Warn, _) | (DownloadBehavior::Error, true) => { - warn!( - "unexpectedly on-demand downloading remote layer {} for task kind {:?}", - id, - ctx.task_kind() - ); - UNEXPECTED_ONDEMAND_DOWNLOADS.inc(); - timeline.download_remote_layer(remote_layer).await?; - continue 'layer_map_search; - } - (DownloadBehavior::Error, false) => { - return Err(PageReconstructError::NeedsDownload( - TenantTimelineId::new(self.tenant_id, self.timeline_id), - remote_layer.filename(), - )) - } + // Check the open and frozen in-memory layers first, in order from newest + // to oldest. + if let Some(open_layer) = &layers.open_layer { + let start_lsn = open_layer.get_lsn_range().start; + if cont_lsn > start_lsn { + //info!("CHECKING for {} at {} on open layer {}", key, cont_lsn, open_layer.filename().display()); + // Get all the data needed to reconstruct the page version from this layer. + // But if we have an older cached page image, no need to go past that. + let lsn_floor = max(cached_lsn + 1, start_lsn); + result = match open_layer + .get_value_reconstruct_data( + key, + lsn_floor..cont_lsn, + reconstruct_state, + ctx, + ) + .await + { + Ok(result) => result, + Err(e) => return Err(PageReconstructError::from(e)), + }; + cont_lsn = lsn_floor; + // metrics: open_layer does not count as fs access, so we are not updating `read_count` + traversal_path.push(( + result, + cont_lsn, + Box::new({ + let open_layer = Arc::clone(open_layer); + move || open_layer.traversal_id() + }), + )); + continue 'outer; } } + for frozen_layer in layers.frozen_layers.iter().rev() { + let start_lsn = frozen_layer.get_lsn_range().start; + if cont_lsn > start_lsn { + //info!("CHECKING for {} at {} on frozen layer {}", key, cont_lsn, frozen_layer.filename().display()); + let lsn_floor = max(cached_lsn + 1, start_lsn); + result = match frozen_layer + .get_value_reconstruct_data( + key, + lsn_floor..cont_lsn, + reconstruct_state, + ctx, + ) + .await + { + Ok(result) => result, + Err(e) => return Err(PageReconstructError::from(e)), + }; + cont_lsn = lsn_floor; + // metrics: open_layer does not count as fs access, so we are not updating `read_count` + traversal_path.push(( + result, + cont_lsn, + Box::new({ + let frozen_layer = Arc::clone(frozen_layer); + move || frozen_layer.traversal_id() + }), + )); + continue 'outer; + } + } + + if let Some(SearchResult { lsn_floor, layer }) = layers.search(key, cont_lsn) { + let layer = guard.get_from_desc(&layer); + // Get all the data needed to reconstruct the page version from this layer. + // But if we have an older cached page image, no need to go past that. + let lsn_floor = max(cached_lsn + 1, lsn_floor); + result = match layer + .get_value_reconstruct_data(key, lsn_floor..cont_lsn, reconstruct_state, ctx) + .await + { + Ok(result) => result, + Err(e) => return Err(PageReconstructError::from(e)), + }; + cont_lsn = lsn_floor; + *read_count += 1; + traversal_path.push(( + result, + cont_lsn, + Box::new({ + let layer = layer.to_owned(); + move || layer.traversal_id() + }), + )); + continue 'outer; + } else if timeline.ancestor_timeline.is_some() { + // Nothing on this timeline. Traverse to parent + result = ValueReconstructResult::Continue; + cont_lsn = Lsn(timeline.ancestor_lsn.0 + 1); + continue 'outer; + } else { + // Nothing found + result = ValueReconstructResult::Missing; + continue 'outer; + } } } @@ -2742,7 +2427,7 @@ impl Timeline { // files instead. This is possible as long as *all* the data imported into the // repository have the same LSN. let lsn_range = frozen_layer.get_lsn_range(); - let (layer_paths_to_upload, delta_layer_to_add) = + let (layers_to_upload, delta_layer_to_add) = if lsn_range.start == self.initdb_lsn && lsn_range.end == Lsn(self.initdb_lsn.0 + 1) { #[cfg(test)] match &mut *self.flush_loop_state.lock().unwrap() { @@ -2785,10 +2470,9 @@ impl Timeline { // We will remove frozen layer and add delta layer in one atomic operation later. let layer = self.create_delta_layer(&frozen_layer, ctx).await?; ( - HashMap::from([( - layer.filename(), - LayerFileMetadata::new(layer.layer_desc().file_size, self.generation), - )]), + // FIXME: even though we have a single image and single delta layer assumption + // we push them to vec + vec![layer.clone()], Some(layer), ) }; @@ -2802,25 +2486,14 @@ impl Timeline { let metadata = { let mut guard = self.layers.write().await; - if let Some(ref l) = delta_layer_to_add { - // TODO: move access stats, metrics update, etc. into layer manager. - l.access_stats().record_residence_event( - LayerResidenceStatus::Resident, - LayerResidenceEventReason::LayerCreate, - ); + guard.finish_flush_l0_layer(delta_layer_to_add.as_ref(), &frozen_layer, &self.metrics); - // update metrics - let sz = l.layer_desc().file_size; - self.metrics.record_new_file_metrics(sz); - } - - guard.finish_flush_l0_layer(delta_layer_to_add, &frozen_layer); if disk_consistent_lsn != old_disk_consistent_lsn { assert!(disk_consistent_lsn > old_disk_consistent_lsn); self.disk_consistent_lsn.store(disk_consistent_lsn); // Schedule remote uploads that will reflect our new disk_consistent_lsn - Some(self.schedule_uploads(disk_consistent_lsn, layer_paths_to_upload)?) + Some(self.schedule_uploads(disk_consistent_lsn, layers_to_upload)?) } else { None } @@ -2855,7 +2528,7 @@ impl Timeline { fn schedule_uploads( &self, disk_consistent_lsn: Lsn, - layer_paths_to_upload: HashMap, + layers_to_upload: impl IntoIterator, ) -> anyhow::Result { // We can only save a valid 'prev_record_lsn' value on disk if we // flushed *all* in-memory changes to disk. We only track @@ -2894,8 +2567,8 @@ impl Timeline { )); if let Some(remote_client) = &self.remote_client { - for (path, layer_metadata) in layer_paths_to_upload { - remote_client.schedule_layer_file_upload(&path, &layer_metadata)?; + for layer in layers_to_upload { + remote_client.schedule_layer_file_upload(layer)?; } remote_client.schedule_index_upload_for_metadata_update(&metadata)?; } @@ -2906,9 +2579,9 @@ impl Timeline { async fn update_metadata_file( &self, disk_consistent_lsn: Lsn, - layer_paths_to_upload: HashMap, + layers_to_upload: impl IntoIterator, ) -> anyhow::Result<()> { - let metadata = self.schedule_uploads(disk_consistent_lsn, layer_paths_to_upload)?; + let metadata = self.schedule_uploads(disk_consistent_lsn, layers_to_upload)?; save_metadata(self.conf, &self.tenant_id, &self.timeline_id, &metadata) .await @@ -2923,10 +2596,9 @@ impl Timeline { self: &Arc, frozen_layer: &Arc, ctx: &RequestContext, - ) -> anyhow::Result { + ) -> anyhow::Result { let span = tracing::info_span!("blocking"); - let new_delta: DeltaLayer = tokio::task::spawn_blocking({ - let _g = span.entered(); + let new_delta: ResidentLayer = tokio::task::spawn_blocking({ let self_clone = Arc::clone(self); let frozen_layer = Arc::clone(frozen_layer); let ctx = ctx.attached_child(); @@ -2936,8 +2608,10 @@ impl Timeline { // as long as the write path is still sync and the read impl // is still not fully async. Otherwise executor threads would // be blocked. - let new_delta = Handle::current().block_on(frozen_layer.write_to_disk(&ctx))?; - let new_delta_path = new_delta.path(); + let _g = span.entered(); + let new_delta = + Handle::current().block_on(frozen_layer.write_to_disk(&self_clone, &ctx))?; + let new_delta_path = new_delta.local_path().to_owned(); // Sync it to disk. // @@ -2954,6 +2628,8 @@ impl Timeline { // 3. rename to the final name // 4. fsync the parent directory. // Note that (1),(2),(3) today happen inside write_to_disk(). + // + // FIXME: the writer already fsyncs all data, only rename needs to be fsynced here par_fsync::par_fsync(&[new_delta_path]).context("fsync of delta layer")?; par_fsync::par_fsync(&[self_clone .conf @@ -2964,7 +2640,8 @@ impl Timeline { } }) .await - .context("spawn_blocking")??; + .context("spawn_blocking") + .and_then(|x| x)?; Ok(new_delta) } @@ -3081,14 +2758,14 @@ impl Timeline { #[tracing::instrument(skip_all, fields(%lsn, %force))] async fn create_image_layers( - &self, + self: &Arc, partitioning: &KeyPartitioning, lsn: Lsn, force: bool, ctx: &RequestContext, - ) -> Result, PageReconstructError> { + ) -> Result, PageReconstructError> { let timer = self.metrics.create_images_time_histo.start_timer(); - let mut image_layers: Vec = Vec::new(); + let mut image_layers = Vec::new(); // We need to avoid holes between generated image layers. // Otherwise LayerMap::image_layer_exists will return false if key range of some layer is covered by more than one @@ -3152,7 +2829,7 @@ impl Timeline { key = key.next(); } } - let image_layer = image_layer_writer.finish().await?; + let image_layer = image_layer_writer.finish(self).await?; image_layers.push(image_layer); } } @@ -3174,7 +2851,7 @@ impl Timeline { // and fsync them all in parallel. let all_paths = image_layers .iter() - .map(|layer| layer.path()) + .map(|layer| layer.local_path().to_owned()) .collect::>(); par_fsync::par_fsync_async(&all_paths) @@ -3185,63 +2862,32 @@ impl Timeline { .await .context("fsync of timeline dir")?; - let mut layer_paths_to_upload = HashMap::with_capacity(image_layers.len()); - let mut guard = self.layers.write().await; - let timeline_path = self.conf.timeline_path(&self.tenant_id, &self.timeline_id); - for l in &image_layers { - let path = l.filename(); - let metadata = timeline_path - .join(path.file_name()) - .metadata() - .with_context(|| format!("reading metadata of layer file {}", path.file_name()))?; - - layer_paths_to_upload.insert( - path, - LayerFileMetadata::new(metadata.len(), self.generation), - ); - - // update metrics - self.metrics.record_new_file_metrics(metadata.len()); - let l = Arc::new(l); - l.access_stats().record_residence_event( - LayerResidenceStatus::Resident, - LayerResidenceEventReason::LayerCreate, - ); - } - guard.track_new_image_layers(image_layers); + // FIXME: we could add the images to be uploaded *before* returning from here, but right + // now they are being scheduled outside of write lock + guard.track_new_image_layers(&image_layers, &self.metrics); drop_wlock(guard); timer.stop_and_record(); - Ok(layer_paths_to_upload) + Ok(image_layers) } } #[derive(Default)] struct CompactLevel0Phase1Result { - new_layers: Vec>, - deltas_to_compact: Vec>, + new_layers: Vec, + deltas_to_compact: Vec, } /// Top-level failure to compact. -#[derive(Debug)] -enum CompactionError { - /// L0 compaction requires layers to be downloaded. - /// - /// This should not happen repeatedly, but will be retried once by top-level - /// `Timeline::compact`. - DownloadRequired(Vec>), - /// The timeline or pageserver is shutting down +#[derive(Debug, thiserror::Error)] +pub(crate) enum CompactionError { + #[error("The timeline or pageserver is shutting down")] ShuttingDown, /// Compaction cannot be done right now; page reconstruction and so on. - Other(anyhow::Error), -} - -impl From for CompactionError { - fn from(value: anyhow::Error) -> Self { - CompactionError::Other(value) - } + #[error(transparent)] + Other(#[from] anyhow::Error), } #[serde_as] @@ -3366,13 +3012,11 @@ impl TryFrom for CompactLevel0Phase1Stats { } impl Timeline { - /// Level0 files first phase of compaction, explained in the [`compact_inner`] comment. + /// Level0 files first phase of compaction, explained in the [`Self::compact`] comment. /// /// This method takes the `_layer_removal_cs` guard to highlight it required downloads are /// returned as an error. If the `layer_removal_cs` boundary is changed not to be taken in the /// start of level0 files compaction, the on-demand download should be revisited as well. - /// - /// [`compact_inner`]: Self::compact_inner async fn compact_level0_phase1( self: &Arc, _layer_removal_cs: Arc>, @@ -3417,19 +3061,30 @@ impl Timeline { // size length. Compaction will likely create the same set of n files afterwards. // // This failpoint is a superset of both of the cases. - fail_point!("compact-level0-phase1-return-same", |_| { - println!("compact-level0-phase1-return-same"); // so that we can check if we hit the failpoint - Ok(CompactLevel0Phase1Result { - new_layers: level0_deltas - .iter() - .map(|x| x.clone().downcast_delta_layer().unwrap()) - .collect(), - deltas_to_compact: level0_deltas - .iter() - .map(|x| x.layer_desc().clone().into()) - .collect(), - }) - }); + if cfg!(feature = "testing") { + let active = (|| { + ::fail::fail_point!("compact-level0-phase1-return-same", |_| true); + false + })(); + + if active { + let mut new_layers = Vec::with_capacity(level0_deltas.len()); + for delta in &level0_deltas { + // we are just faking these layers as being produced again for this failpoint + new_layers.push( + delta + .download_and_keep_resident() + .await + .context("download layer for failpoint")?, + ); + } + tracing::info!("compact-level0-phase1-return-same"); // so that we can check if we hit the failpoint + return Ok(CompactLevel0Phase1Result { + new_layers, + deltas_to_compact: level0_deltas, + }); + } + } // Gather the files to compact in this iteration. // @@ -3449,14 +3104,18 @@ impl Timeline { let first_level0_delta = level0_deltas_iter.next().unwrap(); let mut prev_lsn_end = first_level0_delta.layer_desc().lsn_range.end; - let mut deltas_to_compact = vec![Arc::clone(first_level0_delta)]; + let mut deltas_to_compact = Vec::with_capacity(level0_deltas.len()); + + // FIXME: downloading while holding layer_removal_cs is not great, but we will remove that + // soon + deltas_to_compact.push(first_level0_delta.download_and_keep_resident().await?); for l in level0_deltas_iter { let lsn_range = &l.layer_desc().lsn_range; if lsn_range.start != prev_lsn_end { break; } - deltas_to_compact.push(Arc::clone(l)); + deltas_to_compact.push(l.download_and_keep_resident().await?); prev_lsn_end = lsn_range.end; } let lsn_range = Range { @@ -3469,24 +3128,6 @@ impl Timeline { end: deltas_to_compact.last().unwrap().layer_desc().lsn_range.end, }; - let remotes = deltas_to_compact - .iter() - .filter(|l| l.is_remote_layer()) - .inspect(|l| info!("compact requires download of {l}")) - .map(|l| { - l.clone() - .downcast_remote_layer() - .expect("just checked it is remote layer") - }) - .collect::>(); - - if !remotes.is_empty() { - // caller is holding the lock to layer_removal_cs, and we don't want to download while - // holding that; in future download_remote_layer might take it as well. this is - // regardless of earlier image creation downloading on-demand, while holding the lock. - return Err(CompactionError::DownloadRequired(remotes)); - } - info!( "Starting Level0 compaction in LSN range {}-{} for {} layers ({} deltas in total)", lsn_range.start, @@ -3519,23 +3160,19 @@ impl Timeline { let mut all_keys = Vec::new(); - let downcast_deltas: Vec<_> = deltas_to_compact - .iter() - .map(|l| l.clone().downcast_delta_layer().expect("delta layer")) - .collect(); - for dl in downcast_deltas.iter() { - // TODO: replace this with an await once we fully go async - all_keys.extend(DeltaLayer::load_keys(dl, ctx).await?); + for l in deltas_to_compact.iter() { + all_keys.extend(l.load_keys(ctx).await?); } + // FIXME: should spawn_blocking the rest of this function + // The current stdlib sorting implementation is designed in a way where it is // particularly fast where the slice is made up of sorted sub-ranges. all_keys.sort_by_key(|DeltaEntry { key, lsn, .. }| (*key, *lsn)); stats.read_lock_held_key_sort_micros = stats.read_lock_held_prerequisites_micros.till_now(); - for DeltaEntry { key: next_key, .. } in all_keys.iter() { - let next_key = *next_key; + for &DeltaEntry { key: next_key, .. } in all_keys.iter() { if let Some(prev_key) = prev { // just first fast filter if next_key.to_i128() - prev_key.to_i128() >= min_hole_range { @@ -3695,13 +3332,13 @@ impl Timeline { || contains_hole { // ... if so, flush previous layer and prepare to write new one - new_layers.push(Arc::new( + new_layers.push( writer .take() .unwrap() - .finish(prev_key.unwrap().next()) + .finish(prev_key.unwrap().next(), self) .await?, - )); + ); writer = None; if contains_hole { @@ -3749,7 +3386,7 @@ impl Timeline { prev_key = Some(key); } if let Some(writer) = writer { - new_layers.push(Arc::new(writer.finish(prev_key.unwrap().next()).await?)); + new_layers.push(writer.finish(prev_key.unwrap().next(), self).await?); } // Sync layers @@ -3767,10 +3404,17 @@ impl Timeline { ); } } - let mut layer_paths: Vec = new_layers.iter().map(|l| l.path()).collect(); + + // FIXME: the writer already fsyncs all data, only rename needs to be fsynced here + let mut layer_paths: Vec = new_layers + .iter() + .map(|l| l.local_path().to_owned()) + .collect(); // Fsync all the layer files and directory using multiple threads to // minimize latency. + // + // FIXME: spawn_blocking above for this par_fsync::par_fsync(&layer_paths).context("fsync all new layers")?; par_fsync::par_fsync(&[self.conf.timeline_path(&self.tenant_id, &self.timeline_id)]) @@ -3801,8 +3445,8 @@ impl Timeline { new_layers, deltas_to_compact: deltas_to_compact .into_iter() - .map(|x| Arc::new(x.layer_desc().clone())) - .collect(), + .map(|x| x.drop_eviction_guard()) + .collect::>(), }) } @@ -3863,74 +3507,45 @@ impl Timeline { } let mut guard = self.layers.write().await; - let mut new_layer_paths = HashMap::with_capacity(new_layers.len()); - // In some rare cases, we may generate a file with exactly the same key range / LSN as before the compaction. - // We should move to numbering the layer files instead of naming them using key range / LSN some day. But for - // now, we just skip the file to avoid unintentional modification to files on the disk and in the layer map. let mut duplicated_layers = HashSet::new(); - let mut uploaded_layers = Vec::with_capacity(new_layers.len()); - let mut insert_layers = Vec::new(); - let mut remove_layers = Vec::new(); + let mut insert_layers = Vec::with_capacity(new_layers.len()); for l in &new_layers { - let new_delta_path = l.path(); - - let metadata = new_delta_path.metadata().with_context(|| { - format!("read file metadata for new created layer {new_delta_path}") - })?; - - uploaded_layers.push(( - l.filename(), - LayerFileMetadata::new(metadata.len(), self.generation), - )); - - // update metrics, including the timeline's physical size - self.metrics.record_new_file_metrics(metadata.len()); - - new_layer_paths.insert( - new_delta_path, - LayerFileMetadata::new(metadata.len(), self.generation), - ); - l.access_stats().record_residence_event( - LayerResidenceStatus::Resident, - LayerResidenceEventReason::LayerCreate, - ); - let l = l.to_owned() as Arc; - if guard.contains(&l) { + if guard.contains(l.as_ref()) { + // expected in tests tracing::error!(layer=%l, "duplicated L1 layer"); + + // good ways to cause a duplicate: we repeatedly error after taking the writelock + // `guard` on self.layers. as of writing this, there are no error returns except + // for compact_level0_phase1 creating an L0, which does not happen in practice + // because we have not implemented L0 => L0 compaction. duplicated_layers.insert(l.layer_desc().key()); + } else if LayerMap::is_l0(l.layer_desc()) { + return Err(CompactionError::Other(anyhow!("compaction generates a L0 layer file as output, which will cause infinite compaction."))); } else { - if LayerMap::is_l0(l.layer_desc()) { - return Err(CompactionError::Other(anyhow!("compaction generates a L0 layer file as output, which will cause infinite compaction."))); - } - insert_layers.push(l); + insert_layers.push(l.clone()); } } - // Now that we have reshuffled the data to set of new delta layers, we can - // delete the old ones - let mut layer_names_to_delete = Vec::with_capacity(deltas_to_compact.len()); - for ldesc in deltas_to_compact { - if duplicated_layers.contains(&ldesc.key()) { - // skip duplicated layers, they will not be removed; we have already overwritten them - // with new layers in the compaction phase 1. - continue; - } - layer_names_to_delete.push(ldesc.filename()); - remove_layers.push(guard.get_from_desc(&ldesc)); - } + let remove_layers = { + let mut deltas_to_compact = deltas_to_compact; + // only remove those inputs which were not outputs + deltas_to_compact.retain(|l| !duplicated_layers.contains(&l.layer_desc().key())); + deltas_to_compact + }; + // deletion will happen later, the layer file manager calls garbage_collect_on_drop guard.finish_compact_l0( - layer_removal_cs, - remove_layers, - insert_layers, + &layer_removal_cs, + &remove_layers, + &insert_layers, &self.metrics, - )?; + ); if let Some(remote_client) = self.remote_client.as_ref() { - remote_client.schedule_compaction_update(&layer_names_to_delete, &uploaded_layers)?; + remote_client.schedule_compaction_update(&remove_layers, &new_layers)?; } drop_wlock(guard); @@ -4235,7 +3850,7 @@ impl Timeline { l.filename(), l.is_incremental(), ); - layers_to_remove.push(Arc::clone(&l)); + layers_to_remove.push(l); } self.wanted_image_layers .lock() @@ -4245,32 +3860,28 @@ impl Timeline { if !layers_to_remove.is_empty() { // Persist the new GC cutoff value in the metadata file, before // we actually remove anything. - self.update_metadata_file(self.disk_consistent_lsn.load(), HashMap::new()) + // + // This does not in fact have any effect as we no longer consider local metadata unless + // running without remote storage. + self.update_metadata_file(self.disk_consistent_lsn.load(), None) .await?; - // Actually delete the layers from disk and remove them from the map. - // (couldn't do this in the loop above, because you cannot modify a collection - // while iterating it. BTreeMap::retain() would be another option) - let mut layer_names_to_delete = Vec::with_capacity(layers_to_remove.len()); let gc_layers = layers_to_remove .iter() .map(|x| guard.get_from_desc(x)) - .collect(); - for doomed_layer in layers_to_remove { - layer_names_to_delete.push(doomed_layer.filename()); - result.layers_removed += 1; + .collect::>(); + + result.layers_removed = gc_layers.len() as u64; + + if let Some(remote_client) = self.remote_client.as_ref() { + remote_client.schedule_gc_update(&gc_layers)?; } - let apply = guard.finish_gc_timeline(layer_removal_cs, gc_layers, &self.metrics)?; + + guard.finish_gc_timeline(&layer_removal_cs, gc_layers); if result.layers_removed != 0 { fail_point!("after-timeline-gc-removed-layers"); } - - if let Some(remote_client) = &self.remote_client { - remote_client.schedule_layer_file_deletion(layer_names_to_delete)?; - } - - apply.flush(); } info!( @@ -4367,155 +3978,15 @@ impl Timeline { } } - /// Download a layer file from remote storage and insert it into the layer map. - /// - /// It's safe to call this function for the same layer concurrently. In that case: - /// - If the layer has already been downloaded, `OK(...)` is returned. - /// - If the layer is currently being downloaded, we wait until that download succeeded / failed. - /// - If it succeeded, we return `Ok(...)`. - /// - If it failed, we or another concurrent caller will initiate a new download attempt. - /// - /// Download errors are classified and retried if appropriate by the underlying RemoteTimelineClient function. - /// It has an internal limit for the maximum number of retries and prints appropriate log messages. - /// If we exceed the limit, it returns an error, and this function passes it through. - /// The caller _could_ retry further by themselves by calling this function again, but _should not_ do it. - /// The reason is that they cannot distinguish permanent errors from temporary ones, whereas - /// the underlying RemoteTimelineClient can. - /// - /// There is no internal timeout or slowness detection. - /// If the caller has a deadline or needs a timeout, they can simply stop polling: - /// we're **cancellation-safe** because the download happens in a separate task_mgr task. - /// So, the current download attempt will run to completion even if we stop polling. - #[instrument(skip_all, fields(layer=%remote_layer))] - pub async fn download_remote_layer( - &self, - remote_layer: Arc, - ) -> anyhow::Result<()> { - span::debug_assert_current_span_has_tenant_and_timeline_id(); - - use std::sync::atomic::Ordering::Relaxed; - - let permit = match Arc::clone(&remote_layer.ongoing_download) - .acquire_owned() - .await - { - Ok(permit) => permit, - Err(_closed) => { - if remote_layer.download_replacement_failure.load(Relaxed) { - // this path will be hit often, in case there are upper retries. however - // hitting this error will prevent a busy loop between get_reconstruct_data and - // download, so an error is prefered. - // - // TODO: we really should poison the timeline, but panicking is not yet - // supported. Related: https://github.com/neondatabase/neon/issues/3621 - anyhow::bail!("an earlier download succeeded but LayerMap::replace failed") - } else { - info!("download of layer has already finished"); - return Ok(()); - } - } - }; - - let (sender, receiver) = tokio::sync::oneshot::channel(); - // Spawn a task so that download does not outlive timeline when we detach tenant / delete timeline. - let self_clone = self.myself.upgrade().expect("timeline is gone"); - task_mgr::spawn( - &tokio::runtime::Handle::current(), - TaskKind::RemoteDownloadTask, - Some(self.tenant_id), - Some(self.timeline_id), - &format!("download layer {}", remote_layer), - false, - async move { - let remote_client = self_clone.remote_client.as_ref().unwrap(); - - // Does retries + exponential back-off internally. - // When this fails, don't layer further retry attempts here. - let result = remote_client - .download_layer_file(&remote_layer.filename(), &remote_layer.layer_metadata) - .await; - - if let Ok(size) = &result { - info!("layer file download finished"); - - // XXX the temp file is still around in Err() case - // and consumes space until we clean up upon pageserver restart. - self_clone.metrics.resident_physical_size_add(*size); - - // Download complete. Replace the RemoteLayer with the corresponding - // Delta- or ImageLayer in the layer map. - let mut guard = self_clone.layers.write().await; - let new_layer = - remote_layer.create_downloaded_layer(&guard, self_clone.conf, *size); - { - let l: Arc = remote_layer.clone(); - let failure = match guard.replace_and_verify(l, new_layer) { - Ok(()) => false, - Err(e) => { - // this is a precondition failure, the layer filename derived - // attributes didn't match up, which doesn't seem likely. - error!("replacing downloaded layer into layermap failed: {e:#?}"); - true - } - }; - - if failure { - // mark the remote layer permanently failed; the timeline is most - // likely unusable after this. sadly we cannot just poison the layermap - // lock with panic, because that would create an issue with shutdown. - // - // this does not change the retry semantics on failed downloads. - // - // use of Relaxed is valid because closing of the semaphore gives - // happens-before and wakes up any waiters; we write this value before - // and any waiters (or would be waiters) will load it after closing - // semaphore. - // - // See: https://github.com/neondatabase/neon/issues/3533 - remote_layer - .download_replacement_failure - .store(true, Relaxed); - } - } - drop_wlock(guard); - - info!("on-demand download successful"); - - // Now that we've inserted the download into the layer map, - // close the semaphore. This will make other waiters for - // this download return Ok(()). - assert!(!remote_layer.ongoing_download.is_closed()); - remote_layer.ongoing_download.close(); - } else { - // Keep semaphore open. We'll drop the permit at the end of the function. - error!( - "layer file download failed: {:?}", - result.as_ref().unwrap_err() - ); - } - - // Don't treat it as an error if the task that triggered the download - // is no longer interested in the result. - sender.send(result.map(|_sz| ())).ok(); - - // In case we failed and there are other waiters, this will make one - // of them retry the download in a new task. - // XXX: This resets the exponential backoff because it's a new call to - // download_layer file. - drop(permit); - - Ok(()) - } - .in_current_span(), - ); - - receiver.await.context("download task cancelled")? - } - - pub async fn spawn_download_all_remote_layers( + pub(crate) async fn spawn_download_all_remote_layers( self: Arc, request: DownloadRemoteLayersTaskSpawnRequest, ) -> Result { + use pageserver_api::models::DownloadRemoteLayersTaskState; + + // this is not really needed anymore; it has tests which really check the return value from + // http api. it would be better not to maintain this anymore. + let mut status_guard = self.download_all_remote_layers_task_info.write().unwrap(); if let Some(st) = &*status_guard { match &st.state { @@ -4574,21 +4045,17 @@ impl Timeline { self: &Arc, request: DownloadRemoteLayersTaskSpawnRequest, ) { - let mut downloads = Vec::new(); - { + use pageserver_api::models::DownloadRemoteLayersTaskState; + + let remaining = { let guard = self.layers.read().await; - let layers = guard.layer_map(); - layers + guard + .layer_map() .iter_historic_layers() - .map(|l| guard.get_from_desc(&l)) - .filter_map(|l| l.downcast_remote_layer()) - .map(|l| self.download_remote_layer(l)) - .for_each(|dl| downloads.push(dl)) - } - let total_layer_count = downloads.len(); - // limit download concurrency as specified in request - let downloads = futures::stream::iter(downloads); - let mut downloads = downloads.buffer_unordered(request.max_concurrent_downloads.get()); + .map(|desc| guard.get_from_desc(&desc)) + .collect::>() + }; + let total_layer_count = remaining.len(); macro_rules! lock_status { ($st:ident) => { @@ -4611,29 +4078,58 @@ impl Timeline { lock_status!(st); st.total_layer_count = total_layer_count as u64; } + + let mut remaining = remaining.into_iter(); + let mut have_remaining = true; + let mut js = tokio::task::JoinSet::new(); + + let cancel = task_mgr::shutdown_token(); + + let limit = request.max_concurrent_downloads; + loop { - tokio::select! { - dl = downloads.next() => { - lock_status!(st); - match dl { - None => break, - Some(Ok(())) => { - st.successful_download_count += 1; - }, - Some(Err(e)) => { - error!(error = %e, "layer download failed"); - st.failed_download_count += 1; - } + while js.len() < limit.get() && have_remaining && !cancel.is_cancelled() { + let Some(next) = remaining.next() else { + have_remaining = false; + break; + }; + + let span = tracing::info_span!("download", layer = %next); + + js.spawn( + async move { + let res = next.download().await; + (next, res) } - } - _ = task_mgr::shutdown_watcher() => { - // Kind of pointless to watch for shutdowns here, - // as download_remote_layer spawns other task_mgr tasks internally. - lock_status!(st); - st.state = DownloadRemoteLayersTaskState::ShutDown; + .instrument(span), + ); + } + + while let Some(res) = js.join_next().await { + match res { + Ok((_, Ok(_))) => { + lock_status!(st); + st.successful_download_count += 1; + } + Ok((layer, Err(e))) => { + tracing::error!(%layer, "download failed: {e:#}"); + lock_status!(st); + st.failed_download_count += 1; + } + Err(je) if je.is_cancelled() => unreachable!("not used here"), + Err(je) if je.is_panic() => { + lock_status!(st); + st.failed_download_count += 1; + } + Err(je) => tracing::warn!("unknown joinerror: {je:?}"), } } + + if js.is_empty() && (!have_remaining || cancel.is_cancelled()) { + break; + } } + { lock_status!(st); st.state = DownloadRemoteLayersTaskState::Completed; @@ -4648,15 +4144,15 @@ impl Timeline { } } -pub struct DiskUsageEvictionInfo { +pub(crate) struct DiskUsageEvictionInfo { /// Timeline's largest layer (remote or resident) pub max_layer_size: Option, /// Timeline's resident layers pub resident_layers: Vec, } -pub struct LocalLayerInfoForDiskUsageEviction { - pub layer: Arc, +pub(crate) struct LocalLayerInfoForDiskUsageEviction { + pub layer: Layer, pub last_activity_ts: SystemTime, } @@ -4666,8 +4162,14 @@ impl std::fmt::Debug for LocalLayerInfoForDiskUsageEviction { // having to allocate a string to this is bad, but it will rarely be formatted let ts = chrono::DateTime::::from(self.last_activity_ts); let ts = ts.to_rfc3339_opts(chrono::SecondsFormat::Nanos, true); + struct DisplayIsDebug<'a, T>(&'a T); + impl<'a, T: std::fmt::Display> std::fmt::Debug for DisplayIsDebug<'a, T> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } + } f.debug_struct("LocalLayerInfoForDiskUsageEviction") - .field("layer", &self.layer) + .field("layer", &DisplayIsDebug(&self.layer)) .field("last_activity", &ts) .finish() } @@ -4694,9 +4196,16 @@ impl Timeline { let l = guard.get_from_desc(&l); - if l.is_remote_layer() { - continue; - } + let l = match l.keep_resident().await { + Ok(Some(l)) => l, + Ok(None) => continue, + Err(e) => { + // these should not happen, but we cannot make them statically impossible right + // now. + tracing::warn!(layer=%l, "failed to keep the layer resident: {e:#}"); + continue; + } + }; let last_activity_ts = l.access_stats().latest_activity().unwrap_or_else(|| { // We only use this fallback if there's an implementation error. @@ -4706,7 +4215,7 @@ impl Timeline { }); resident_layers.push(LocalLayerInfoForDiskUsageEviction { - layer: l, + layer: l.drop_eviction_guard(), last_activity_ts, }); } @@ -4829,43 +4338,13 @@ fn rename_to_backup(path: &Utf8Path) -> anyhow::Result<()> { bail!("couldn't find an unused backup number for {:?}", path) } -/// Similar to `Arc::ptr_eq`, but only compares the object pointers, not vtables. -/// -/// Returns `true` if the two `Arc` point to the same layer, false otherwise. -/// -/// If comparing persistent layers, ALWAYS compare the layer descriptor key. -#[inline(always)] -pub fn compare_arced_layers(left: &Arc, right: &Arc) -> bool { - // "dyn Trait" objects are "fat pointers" in that they have two components: - // - pointer to the object - // - pointer to the vtable - // - // rust does not provide a guarantee that these vtables are unique, but however - // `Arc::ptr_eq` as of writing (at least up to 1.67) uses a comparison where both the - // pointer and the vtable need to be equal. - // - // See: https://github.com/rust-lang/rust/issues/103763 - // - // A future version of rust will most likely use this form below, where we cast each - // pointer into a pointer to unit, which drops the inaccessible vtable pointer, making it - // not affect the comparison. - // - // See: https://github.com/rust-lang/rust/pull/106450 - let left = Arc::as_ptr(left) as *const (); - let right = Arc::as_ptr(right) as *const (); - - left == right -} - #[cfg(test)] mod tests { - use std::sync::Arc; - use utils::{id::TimelineId, lsn::Lsn}; - use crate::tenant::{harness::TenantHarness, storage_layer::PersistentLayer}; - - use super::{EvictionError, Timeline}; + use crate::tenant::{ + harness::TenantHarness, storage_layer::Layer, timeline::EvictionError, Timeline, + }; #[tokio::test] async fn two_layer_eviction_attempts_at_the_same_time() { @@ -4885,22 +4364,29 @@ mod tests { .expect("just configured this"); let layer = find_some_layer(&timeline).await; + let layer = layer + .keep_resident() + .await + .expect("no download => no downloading errors") + .expect("should had been resident") + .drop_eviction_guard(); let cancel = tokio_util::sync::CancellationToken::new(); let batch = [layer]; let first = { - let cancel = cancel.clone(); + let cancel = cancel.child_token(); async { + let cancel = cancel; timeline - .evict_layer_batch(&rc, &batch, cancel) + .evict_layer_batch(&rc, &batch, &cancel) .await .unwrap() } }; let second = async { timeline - .evict_layer_batch(&rc, &batch, cancel) + .evict_layer_batch(&rc, &batch, &cancel) .await .unwrap() }; @@ -4909,76 +4395,21 @@ mod tests { let (first, second) = (only_one(first), only_one(second)); + let res = batch[0].keep_resident().await; + assert!(matches!(res, Ok(None)), "{res:?}"); + match (first, second) { - (Ok(()), Err(EvictionError::FileNotFound)) - | (Err(EvictionError::FileNotFound), Ok(())) => { - // one of the evictions gets to do it, - // other one gets FileNotFound. all is good. + (Ok(()), Ok(())) => { + // because there are no more timeline locks being taken on eviction path, we can + // witness all three outcomes here. + } + (Ok(()), Err(EvictionError::NotFound)) | (Err(EvictionError::NotFound), Ok(())) => { + // if one completes before the other, this is fine just as well. } other => unreachable!("unexpected {:?}", other), } } - #[tokio::test] - async fn layer_eviction_aba_fails() { - let harness = TenantHarness::create("layer_eviction_aba_fails").unwrap(); - - let ctx = any_context(); - let tenant = harness.try_load(&ctx).await.unwrap(); - let timeline = tenant - .create_test_timeline(TimelineId::generate(), Lsn(0x10), 14, &ctx) - .await - .unwrap(); - - let _e = tracing::info_span!("foobar", tenant_id = %tenant.tenant_id, timeline_id = %timeline.timeline_id).entered(); - - let rc = timeline.remote_client.clone().unwrap(); - - // TenantHarness allows uploads to happen given GenericRemoteStorage is configured - let layer = find_some_layer(&timeline).await; - - let cancel = tokio_util::sync::CancellationToken::new(); - let batch = [layer]; - - let first = { - let cancel = cancel.clone(); - async { - timeline - .evict_layer_batch(&rc, &batch, cancel) - .await - .unwrap() - } - }; - - // lets imagine this is stuck somehow, still referencing the original `Arc` - let second = { - let cancel = cancel.clone(); - async { - timeline - .evict_layer_batch(&rc, &batch, cancel) - .await - .unwrap() - } - }; - - // while it's stuck, we evict and end up redownloading it - only_one(first.await).expect("eviction succeeded"); - - let layer = find_some_layer(&timeline).await; - let layer = layer.downcast_remote_layer().unwrap(); - timeline.download_remote_layer(layer).await.unwrap(); - - let res = only_one(second.await); - - assert!( - matches!(res, Err(EvictionError::LayerNotFound(_))), - "{res:?}" - ); - - // no more specific asserting, outside of preconds this is the only valid replacement - // failure - } - fn any_context() -> crate::context::RequestContext { use crate::context::*; use crate::task_mgr::*; @@ -4993,7 +4424,7 @@ mod tests { .expect("no cancellation") } - async fn find_some_layer(timeline: &Timeline) -> Arc { + async fn find_some_layer(timeline: &Timeline) -> Layer { let layers = timeline.layers.read().await; let desc = layers .layer_map() diff --git a/pageserver/src/tenant/timeline/eviction_task.rs b/pageserver/src/tenant/timeline/eviction_task.rs index 38da993deb..dc5c71bbe1 100644 --- a/pageserver/src/tenant/timeline/eviction_task.rs +++ b/pageserver/src/tenant/timeline/eviction_task.rs @@ -29,7 +29,6 @@ use crate::{ task_mgr::{self, TaskKind, BACKGROUND_RUNTIME}, tenant::{ config::{EvictionPolicy, EvictionPolicyLayerAccessThreshold}, - storage_layer::PersistentLayer, tasks::{BackgroundLoopKind, RateLimitError}, timeline::EvictionError, LogicalSizeCalculationCause, Tenant, @@ -210,15 +209,26 @@ impl Timeline { // NB: all the checks can be invalidated as soon as we release the layer map lock. // We don't want to hold the layer map lock during eviction. // So, we just need to deal with this. - let candidates: Vec> = { + let candidates: Vec<_> = { let guard = self.layers.read().await; let layers = guard.layer_map(); let mut candidates = Vec::new(); for hist_layer in layers.iter_historic_layers() { let hist_layer = guard.get_from_desc(&hist_layer); - if hist_layer.is_remote_layer() { - continue; - } + + // guard against eviction while we inspect it; it might be that eviction_task and + // disk_usage_eviction_task both select the same layers to be evicted, and + // seemingly free up double the space. both succeeding is of no consequence. + let guard = match hist_layer.keep_resident().await { + Ok(Some(l)) => l, + Ok(None) => continue, + Err(e) => { + // these should not happen, but we cannot make them statically impossible right + // now. + tracing::warn!(layer=%hist_layer, "failed to keep the layer resident: {e:#}"); + continue; + } + }; let last_activity_ts = hist_layer.access_stats().latest_activity().unwrap_or_else(|| { // We only use this fallback if there's an implementation error. @@ -249,7 +259,7 @@ impl Timeline { } }; if no_activity_for > p.threshold { - candidates.push(hist_layer) + candidates.push(guard.drop_eviction_guard()) } } candidates @@ -268,7 +278,7 @@ impl Timeline { }; let results = match self - .evict_layer_batch(remote_client, &candidates[..], cancel.clone()) + .evict_layer_batch(remote_client, &candidates, cancel) .await { Err(pre_err) => { @@ -279,7 +289,7 @@ impl Timeline { Ok(results) => results, }; assert_eq!(results.len(), candidates.len()); - for (l, result) in candidates.iter().zip(results) { + for result in results { match result { None => { stats.skipped_for_shutdown += 1; @@ -287,24 +297,10 @@ impl Timeline { Some(Ok(())) => { stats.evicted += 1; } - Some(Err(EvictionError::CannotEvictRemoteLayer)) => { - stats.not_evictable += 1; - } - Some(Err(EvictionError::FileNotFound)) => { + Some(Err(EvictionError::NotFound | EvictionError::Downloaded)) => { // compaction/gc removed the file while we were waiting on layer_removal_cs stats.not_evictable += 1; } - Some(Err( - e @ EvictionError::LayerNotFound(_) | e @ EvictionError::StatFailed(_), - )) => { - let e = utils::error::report_compact_sources(&e); - warn!(layer = %l, "failed to evict layer: {e}"); - stats.not_evictable += 1; - } - Some(Err(EvictionError::MetadataInconsistency(detail))) => { - warn!(layer = %l, "failed to evict layer: {detail}"); - stats.not_evictable += 1; - } } } if stats.candidates == stats.not_evictable { diff --git a/pageserver/src/tenant/timeline/layer_manager.rs b/pageserver/src/tenant/timeline/layer_manager.rs index 0a387bd779..e4991e0865 100644 --- a/pageserver/src/tenant/timeline/layer_manager.rs +++ b/pageserver/src/tenant/timeline/layer_manager.rs @@ -12,27 +12,16 @@ use crate::{ tenant::{ layer_map::{BatchedUpdates, LayerMap}, storage_layer::{ - AsLayerDesc, DeltaLayer, ImageLayer, InMemoryLayer, PersistentLayer, - PersistentLayerDesc, PersistentLayerKey, + AsLayerDesc, InMemoryLayer, Layer, PersistentLayerDesc, PersistentLayerKey, + ResidentLayer, }, - timeline::compare_arced_layers, }, }; /// Provides semantic APIs to manipulate the layer map. pub(crate) struct LayerManager { layer_map: LayerMap, - layer_fmgr: LayerFileManager, -} - -/// After GC, the layer map changes will not be applied immediately. Users should manually apply the changes after -/// scheduling deletes in remote client. -pub(crate) struct ApplyGcResultGuard<'a>(BatchedUpdates<'a>); - -impl ApplyGcResultGuard<'_> { - pub(crate) fn flush(self) { - self.0.flush(); - } + layer_fmgr: LayerFileManager, } impl LayerManager { @@ -43,7 +32,7 @@ impl LayerManager { } } - pub(crate) fn get_from_desc(&self, desc: &PersistentLayerDesc) -> Arc { + pub(crate) fn get_from_desc(&self, desc: &PersistentLayerDesc) -> Layer { self.layer_fmgr.get_from_desc(desc) } @@ -55,21 +44,12 @@ impl LayerManager { &self.layer_map } - /// Replace layers in the layer file manager, used in evictions and layer downloads. - pub(crate) fn replace_and_verify( - &mut self, - expected: Arc, - new: Arc, - ) -> Result<()> { - self.layer_fmgr.replace_and_verify(expected, new) - } - /// Called from `load_layer_map`. Initialize the layer manager with: /// 1. all on-disk layers /// 2. next open layer (with disk disk_consistent_lsn LSN) pub(crate) fn initialize_local_layers( &mut self, - on_disk_layers: Vec>, + on_disk_layers: Vec, next_open_layer_at: Lsn, ) { let mut updates = self.layer_map.batch_update(); @@ -164,10 +144,19 @@ impl LayerManager { } /// Add image layers to the layer map, called from `create_image_layers`. - pub(crate) fn track_new_image_layers(&mut self, image_layers: Vec) { + pub(crate) fn track_new_image_layers( + &mut self, + image_layers: &[ResidentLayer], + metrics: &TimelineMetrics, + ) { let mut updates = self.layer_map.batch_update(); for layer in image_layers { - Self::insert_historic_layer(Arc::new(layer), &mut updates, &mut self.layer_fmgr); + Self::insert_historic_layer(layer.as_ref().clone(), &mut updates, &mut self.layer_fmgr); + + // record these here instead of Layer::finish_creating because otherwise partial + // failure with create_image_layers would balloon up the physical size gauge. downside + // is that all layers need to be created before metrics are updated. + metrics.record_new_file_metrics(layer.layer_desc().file_size); } updates.flush(); } @@ -175,76 +164,71 @@ impl LayerManager { /// Flush a frozen layer and add the written delta layer to the layer map. pub(crate) fn finish_flush_l0_layer( &mut self, - delta_layer: Option, + delta_layer: Option<&ResidentLayer>, frozen_layer_for_check: &Arc, + metrics: &TimelineMetrics, ) { - let l = self.layer_map.frozen_layers.pop_front(); - let mut updates = self.layer_map.batch_update(); + let inmem = self + .layer_map + .frozen_layers + .pop_front() + .expect("there must be a inmem layer to flush"); - // Only one thread may call this function at a time (for this - // timeline). If two threads tried to flush the same frozen + // Only one task may call this function at a time (for this + // timeline). If two tasks tried to flush the same frozen // layer to disk at the same time, that would not work. - assert!(compare_arced_layers(&l.unwrap(), frozen_layer_for_check)); + assert_eq!(Arc::as_ptr(&inmem), Arc::as_ptr(frozen_layer_for_check)); - if let Some(delta_layer) = delta_layer { - Self::insert_historic_layer(Arc::new(delta_layer), &mut updates, &mut self.layer_fmgr); + if let Some(l) = delta_layer { + let mut updates = self.layer_map.batch_update(); + Self::insert_historic_layer(l.as_ref().clone(), &mut updates, &mut self.layer_fmgr); + metrics.record_new_file_metrics(l.layer_desc().file_size); + updates.flush(); } - updates.flush(); } /// Called when compaction is completed. pub(crate) fn finish_compact_l0( &mut self, - layer_removal_cs: Arc>, - compact_from: Vec>, - compact_to: Vec>, + layer_removal_cs: &Arc>, + compact_from: &[Layer], + compact_to: &[ResidentLayer], metrics: &TimelineMetrics, - ) -> Result<()> { + ) { let mut updates = self.layer_map.batch_update(); for l in compact_to { - Self::insert_historic_layer(l, &mut updates, &mut self.layer_fmgr); + Self::insert_historic_layer(l.as_ref().clone(), &mut updates, &mut self.layer_fmgr); + metrics.record_new_file_metrics(l.layer_desc().file_size); } for l in compact_from { - // NB: the layer file identified by descriptor `l` is guaranteed to be present - // in the LayerFileManager because compaction kept holding `layer_removal_cs` the entire - // time, even though we dropped `Timeline::layers` inbetween. - Self::delete_historic_layer( - layer_removal_cs.clone(), - l, - &mut updates, - metrics, - &mut self.layer_fmgr, - )?; + Self::delete_historic_layer(layer_removal_cs, l, &mut updates, &mut self.layer_fmgr); } updates.flush(); - Ok(()) } /// Called when garbage collect the timeline. Returns a guard that will apply the updates to the layer map. pub(crate) fn finish_gc_timeline( &mut self, - layer_removal_cs: Arc>, - gc_layers: Vec>, - metrics: &TimelineMetrics, - ) -> Result { + layer_removal_cs: &Arc>, + gc_layers: Vec, + ) { let mut updates = self.layer_map.batch_update(); for doomed_layer in gc_layers { Self::delete_historic_layer( - layer_removal_cs.clone(), - doomed_layer, + layer_removal_cs, + &doomed_layer, &mut updates, - metrics, &mut self.layer_fmgr, - )?; // FIXME: schedule succeeded deletions in timeline.rs `gc_timeline` instead of in batch? + ); } - Ok(ApplyGcResultGuard(updates)) + updates.flush() } /// Helper function to insert a layer into the layer map and file manager. fn insert_historic_layer( - layer: Arc, + layer: Layer, updates: &mut BatchedUpdates<'_>, - mapping: &mut LayerFileManager, + mapping: &mut LayerFileManager, ) { updates.insert_historic(layer.layer_desc().clone()); mapping.insert(layer); @@ -254,17 +238,12 @@ impl LayerManager { /// Remote storage is not affected by this operation. fn delete_historic_layer( // we cannot remove layers otherwise, since gc and compaction will race - _layer_removal_cs: Arc>, - layer: Arc, + _layer_removal_cs: &Arc>, + layer: &Layer, updates: &mut BatchedUpdates<'_>, - metrics: &TimelineMetrics, - mapping: &mut LayerFileManager, - ) -> anyhow::Result<()> { + mapping: &mut LayerFileManager, + ) { let desc = layer.layer_desc(); - if !layer.is_remote_layer() { - layer.delete_resident_layer_file()?; - metrics.resident_physical_size_sub(desc.file_size); - } // TODO Removing from the bottom of the layer map is expensive. // Maybe instead discard all layer map historic versions that @@ -273,21 +252,18 @@ impl LayerManager { // map index without actually rebuilding the index. updates.remove_historic(desc); mapping.remove(layer); - - Ok(()) + layer.garbage_collect_on_drop(); } - pub(crate) fn contains(&self, layer: &Arc) -> bool { + pub(crate) fn contains(&self, layer: &Layer) -> bool { self.layer_fmgr.contains(layer) } } -pub(crate) struct LayerFileManager( - HashMap>, -); +pub(crate) struct LayerFileManager(HashMap); -impl LayerFileManager { - fn get_from_desc(&self, desc: &PersistentLayerDesc) -> Arc { +impl LayerFileManager { + fn get_from_desc(&self, desc: &PersistentLayerDesc) -> T { // The assumption for the `expect()` is that all code maintains the following invariant: // A layer's descriptor is present in the LayerMap => the LayerFileManager contains a layer for the descriptor. self.0 @@ -297,14 +273,14 @@ impl LayerFileManager { .clone() } - pub(crate) fn insert(&mut self, layer: Arc) { + pub(crate) fn insert(&mut self, layer: T) { let present = self.0.insert(layer.layer_desc().key(), layer.clone()); if present.is_some() && cfg!(debug_assertions) { panic!("overwriting a layer: {:?}", layer.layer_desc()) } } - pub(crate) fn contains(&self, layer: &Arc) -> bool { + pub(crate) fn contains(&self, layer: &T) -> bool { self.0.contains_key(&layer.layer_desc().key()) } @@ -312,7 +288,7 @@ impl LayerFileManager { Self(HashMap::new()) } - pub(crate) fn remove(&mut self, layer: Arc) { + pub(crate) fn remove(&mut self, layer: &T) { let present = self.0.remove(&layer.layer_desc().key()); if present.is_none() && cfg!(debug_assertions) { panic!( @@ -321,39 +297,4 @@ impl LayerFileManager { ) } } - - pub(crate) fn replace_and_verify(&mut self, expected: Arc, new: Arc) -> Result<()> { - let key = expected.layer_desc().key(); - let other = new.layer_desc().key(); - - let expected_l0 = LayerMap::is_l0(expected.layer_desc()); - let new_l0 = LayerMap::is_l0(new.layer_desc()); - - fail::fail_point!("layermap-replace-notfound", |_| anyhow::bail!( - "layermap-replace-notfound" - )); - - anyhow::ensure!( - key == other, - "expected and new layer have different keys: {key:?} != {other:?}" - ); - - anyhow::ensure!( - expected_l0 == new_l0, - "one layer is l0 while the other is not: {expected_l0} != {new_l0}" - ); - - if let Some(layer) = self.0.get_mut(&key) { - anyhow::ensure!( - compare_arced_layers(&expected, layer), - "another layer was found instead of expected, expected={expected:?}, new={new:?}", - expected = Arc::as_ptr(&expected), - new = Arc::as_ptr(layer), - ); - *layer = new; - Ok(()) - } else { - anyhow::bail!("layer was not found"); - } - } } diff --git a/pageserver/src/tenant/upload_queue.rs b/pageserver/src/tenant/upload_queue.rs index 2db67d071a..5959819548 100644 --- a/pageserver/src/tenant/upload_queue.rs +++ b/pageserver/src/tenant/upload_queue.rs @@ -1,4 +1,5 @@ use super::storage_layer::LayerFileName; +use super::storage_layer::ResidentLayer; use super::Generation; use crate::tenant::metadata::TimelineMetadata; use crate::tenant::remote_timeline_client::index::IndexPart; @@ -203,18 +204,6 @@ impl UploadQueue { UploadQueue::Stopped(stopped) => Ok(stopped), } } - - pub(crate) fn get_layer_metadata( - &self, - name: &LayerFileName, - ) -> anyhow::Result> { - match self { - UploadQueue::Stopped(_) | UploadQueue::Uninitialized => { - anyhow::bail!("queue is in state {}", self.as_str()) - } - UploadQueue::Initialized(inner) => Ok(inner.latest_files.get(name).cloned()), - } - } } /// An in-progress upload or delete task. @@ -237,7 +226,7 @@ pub(crate) struct Delete { #[derive(Debug)] pub(crate) enum UploadOp { /// Upload a layer file - UploadLayer(LayerFileName, LayerFileMetadata), + UploadLayer(ResidentLayer, LayerFileMetadata), /// Upload the metadata file UploadMetadata(IndexPart, Lsn), @@ -252,13 +241,13 @@ pub(crate) enum UploadOp { impl std::fmt::Display for UploadOp { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { - UploadOp::UploadLayer(path, metadata) => { + UploadOp::UploadLayer(layer, metadata) => { write!( f, "UploadLayer({}, size={:?}, gen={:?})", - path.file_name(), + layer, metadata.file_size(), - metadata.generation, + metadata.generation ) } UploadOp::UploadMetadata(_, lsn) => { diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 1a7db18b07..0c0c06aa86 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1616,7 +1616,7 @@ class NeonPageserver(PgProtocol): ".*wait for layer upload ops to complete.*", # .*Caused by:.*wait_completion aborted because upload queue was stopped ".*gc_loop.*Gc failed, retrying in.*timeline is Stopping", # When gc checks timeline state after acquiring layer_removal_cs ".*gc_loop.*Gc failed, retrying in.*: Cannot run GC iteration on inactive tenant", # Tenant::gc precondition - ".*compaction_loop.*Compaction failed, retrying in.*timeline is Stopping", # When compaction checks timeline state after acquiring layer_removal_cs + ".*compaction_loop.*Compaction failed, retrying in.*timeline or pageserver is shutting down", # When compaction checks timeline state after acquiring layer_removal_cs ".*query handler for 'pagestream.*failed: Timeline .* was not found", # postgres reconnects while timeline_delete doesn't hold the tenant's timelines.lock() ".*query handler for 'pagestream.*failed: Timeline .* is not active", # timeline delete in progress ".*task iteration took longer than the configured period.*", diff --git a/test_runner/regress/test_attach_tenant_config.py b/test_runner/regress/test_attach_tenant_config.py index 892e40b19b..10cffb1d6c 100644 --- a/test_runner/regress/test_attach_tenant_config.py +++ b/test_runner/regress/test_attach_tenant_config.py @@ -19,7 +19,7 @@ def positive_env(neon_env_builder: NeonEnvBuilder) -> NeonEnv: # eviction might be the first one after an attach to access the layers env.pageserver.allowed_errors.append( - ".*unexpectedly on-demand downloading remote layer remote.* for task kind Eviction" + ".*unexpectedly on-demand downloading remote layer .* for task kind Eviction" ) assert isinstance(env.pageserver_remote_storage, LocalFsStorage) return env diff --git a/test_runner/regress/test_broken_timeline.py b/test_runner/regress/test_broken_timeline.py index c80f2d8360..48accdb43d 100644 --- a/test_runner/regress/test_broken_timeline.py +++ b/test_runner/regress/test_broken_timeline.py @@ -20,7 +20,7 @@ def test_broken_timeline(neon_env_builder: NeonEnvBuilder): env.pageserver.allowed_errors.extend( [ - ".*Failed to load delta layer.*", + ".*layer loading failed:.*", ".*could not find data for key.*", ".*is not active. Current state: Broken.*", ".*will not become active. Current state: Broken.*", @@ -87,7 +87,7 @@ def test_broken_timeline(neon_env_builder: NeonEnvBuilder): # Second timeline will fail during basebackup, because the local layer file is corrupt. # It will fail when we try to read (and reconstruct) a page from it, ergo the error message. # (We don't check layer file contents on startup, when loading the timeline) - with pytest.raises(Exception, match="Failed to load delta layer") as err: + with pytest.raises(Exception, match="layer loading failed:") as err: pg2.start() log.info( f"As expected, compute startup failed for timeline {tenant2}/{timeline2} with corrupt layers: {err}" diff --git a/test_runner/regress/test_layer_eviction.py b/test_runner/regress/test_layer_eviction.py index 3ab3700866..6a6273760c 100644 --- a/test_runner/regress/test_layer_eviction.py +++ b/test_runner/regress/test_layer_eviction.py @@ -247,34 +247,34 @@ def test_gc_of_remote_layers(neon_env_builder: NeonEnvBuilder): ps_http.evict_all_layers(tenant_id, timeline_id) def ensure_resident_and_remote_size_metrics(): - log.info("ensure that all the layers are gone") resident_layers = list(env.pageserver.timeline_dir(tenant_id, timeline_id).glob("*-*_*")) # we have disabled all background loops, so, this should hold - assert len(resident_layers) == 0 + assert len(resident_layers) == 0, "ensure that all the layers are gone" info = ps_http.layer_map_info(tenant_id, timeline_id) log.info("layer map dump: %s", info) - log.info("ensure that resident_physical_size metric is zero") resident_physical_size_metric = ps_http.get_timeline_metric( tenant_id, timeline_id, "pageserver_resident_physical_size" ) - assert resident_physical_size_metric == 0 - log.info("ensure that resident_physical_size metric corresponds to layer map dump") + assert ( + resident_physical_size_metric == 0 + ), "ensure that resident_physical_size metric is zero" assert resident_physical_size_metric == sum( - [layer.layer_file_size or 0 for layer in info.historic_layers if not layer.remote] - ) + layer.layer_file_size or 0 for layer in info.historic_layers if not layer.remote + ), "ensure that resident_physical_size metric corresponds to layer map dump" - log.info("ensure that remote_physical_size metric matches layer map") remote_physical_size_metric = ps_http.get_timeline_metric( tenant_id, timeline_id, "pageserver_remote_physical_size" ) - log.info("ensure that remote_physical_size metric corresponds to layer map dump") assert remote_physical_size_metric == sum( layer.layer_file_size or 0 for layer in info.historic_layers if layer.remote - ) + ), "ensure that remote_physical_size metric corresponds to layer map dump" log.info("before runnning GC, ensure that remote_physical size is zero") + # leaving index_part.json upload from successful compaction out will show + # up here as a mismatch between remove_physical_size and summed up layermap + # size ensure_resident_and_remote_size_metrics() log.info("run GC") diff --git a/test_runner/regress/test_ondemand_download.py b/test_runner/regress/test_ondemand_download.py index 33de7445a4..a4cd42b6c3 100644 --- a/test_runner/regress/test_ondemand_download.py +++ b/test_runner/regress/test_ondemand_download.py @@ -12,13 +12,12 @@ from fixtures.neon_fixtures import ( last_flush_lsn_upload, wait_for_last_flush_lsn, ) -from fixtures.pageserver.http import PageserverApiException, PageserverHttpClient +from fixtures.pageserver.http import PageserverHttpClient from fixtures.pageserver.utils import ( assert_tenant_state, wait_for_last_record_lsn, wait_for_upload, wait_for_upload_queue_empty, - wait_until_tenant_state, ) from fixtures.remote_storage import RemoteStorageKind, available_remote_storages from fixtures.types import Lsn @@ -384,7 +383,7 @@ def test_download_remote_layers_api( env.pageserver.start(extra_env_vars={"FAILPOINTS": "remote-storage-download-pre-rename=return"}) env.pageserver.allowed_errors.extend( [ - f".*download_all_remote_layers.*{tenant_id}.*{timeline_id}.*layer download failed.*remote-storage-download-pre-rename failpoint", + ".*download failed: downloading evicted layer file failed.*", f".*initial size calculation.*{tenant_id}.*{timeline_id}.*Failed to calculate logical size", ] ) @@ -637,56 +636,5 @@ def test_compaction_downloads_on_demand_with_image_creation(neon_env_builder: Ne assert dict(kinds_after) == {"Delta": 4, "Image": 1} -def test_ondemand_download_failure_to_replace(neon_env_builder: NeonEnvBuilder): - """ - Make sure that we fail on being unable to replace a RemoteLayer instead of for example livelocking. - - See: https://github.com/neondatabase/neon/issues/3533 - """ - - neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) - - # disable gc and compaction via default tenant config because config is lost while detaching - # so that compaction will not be the one to download the layer but the http handler is - neon_env_builder.pageserver_config_override = ( - """tenant_config={gc_period = "0s", compaction_period = "0s"}""" - ) - - env = neon_env_builder.init_start() - - tenant_id = env.initial_tenant - timeline_id = env.initial_timeline - assert timeline_id is not None - - pageserver_http = env.pageserver.http_client() - - # remove layers so that they will be redownloaded - pageserver_http.tenant_detach(tenant_id) - pageserver_http.tenant_attach(tenant_id) - - wait_until_tenant_state(pageserver_http, tenant_id, "Active", 5) - pageserver_http.configure_failpoints(("layermap-replace-notfound", "return")) - - # requesting details with non-incremental size should trigger a download of the only layer - # this will need to be adjusted if an index for logical sizes is ever implemented - with pytest.raises(PageserverApiException): - # PageserverApiException is expected because of the failpoint (timeline_detail building does something) - # ReadTimeout can happen on our busy CI, but it should not, because there is no more busylooping - # but should it be added back, we would wait for 15s here. - pageserver_http.timeline_detail(tenant_id, timeline_id, True, timeout=15) - - actual_message = ".* ERROR .*layermap-replace-notfound" - assert env.pageserver.log_contains(actual_message) is not None - env.pageserver.allowed_errors.append(actual_message) - - env.pageserver.allowed_errors.append( - ".* ERROR .*Error processing HTTP request: InternalServerError\\(get local timeline info" - ) - # this might get to run and attempt on-demand, but not always - env.pageserver.allowed_errors.append(".* ERROR .*Task 'initial size calculation'") - - # if the above returned, then we didn't have a livelock, and all is well - - def stringify(conf: Dict[str, Any]) -> Dict[str, str]: return dict(map(lambda x: (x[0], str(x[1])), conf.items())) diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py index f4bf9207b0..f201c735e1 100644 --- a/test_runner/regress/test_remote_storage.py +++ b/test_runner/regress/test_remote_storage.py @@ -586,7 +586,7 @@ def test_timeline_deletion_with_files_stuck_in_upload_queue( log.info("sending delete request") checkpoint_allowed_to_fail.set() env.pageserver.allowed_errors.append( - ".* ERROR .*Error processing HTTP request: InternalServerError\\(timeline is Stopping" + ".* ERROR .*Error processing HTTP request: InternalServerError\\(The timeline or pageserver is shutting down" ".* ERROR .*[Cc]ould not flush frozen layer.*" ) diff --git a/test_runner/regress/test_tenants_with_remote_storage.py b/test_runner/regress/test_tenants_with_remote_storage.py index 30a36d0eca..0169335a70 100644 --- a/test_runner/regress/test_tenants_with_remote_storage.py +++ b/test_runner/regress/test_tenants_with_remote_storage.py @@ -227,9 +227,7 @@ def test_tenant_redownloads_truncated_file_on_startup( assert isinstance(env.pageserver_remote_storage, LocalFsStorage) - env.pageserver.allowed_errors.append( - ".*removing local file .* because it has unexpected length.*" - ) + env.pageserver.allowed_errors.append(".*removing local file .* because .*") # FIXME: Are these expected? env.pageserver.allowed_errors.append( From e0ebdfc7ce82fce9f382031aab738c6d88d3fb23 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 26 Oct 2023 10:59:24 +0100 Subject: [PATCH 24/69] pageserver: suppress compaction/gc errors while stopping (#5670) ## Problem Tenant deletions would sometimes be accompanied by compaction stack traces, because `shutdown()` puts the tenant into stopping state before it joins background tasks. ## Summary of changes Treat GC+Compaction as no-ops on a Stopping tenant. --- pageserver/src/tenant.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index f177a17afc..919291a7e7 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -1822,6 +1822,11 @@ impl Tenant { pitr: Duration, ctx: &RequestContext, ) -> anyhow::Result { + // Don't start doing work during shutdown + if let TenantState::Stopping { .. } = self.current_state() { + return Ok(GcResult::default()); + } + // there is a global allowed_error for this anyhow::ensure!( self.is_active(), @@ -1850,6 +1855,12 @@ impl Tenant { cancel: &CancellationToken, ctx: &RequestContext, ) -> anyhow::Result<()> { + // Don't start doing work during shutdown + if let TenantState::Stopping { .. } = self.current_state() { + return Ok(()); + } + + // We should only be called once the tenant has activated. anyhow::ensure!( self.is_active(), "Cannot run compaction iteration on inactive tenant" From 6129077d31266361a79bf91cc9f95572e8fd39b5 Mon Sep 17 00:00:00 2001 From: MMeent Date: Thu, 26 Oct 2023 13:21:41 +0200 Subject: [PATCH 25/69] WALRedo: Limit logging to log_level = ERROR and above (#5587) This fixes issues in pageserver's walredo process where WALRedo logs of loglevel=LOG are interpreted as errors. ## Problem See #5560 ## Summary of changes Set the log level to something that doesn't include LOG. --- pgxn/neon_walredo/walredoproc.c | 10 ++++++++++ test_runner/fixtures/neon_fixtures.py | 1 - 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pgxn/neon_walredo/walredoproc.c b/pgxn/neon_walredo/walredoproc.c index 01e12983a6..e0c6c6e30a 100644 --- a/pgxn/neon_walredo/walredoproc.c +++ b/pgxn/neon_walredo/walredoproc.c @@ -201,6 +201,16 @@ WalRedoMain(int argc, char *argv[]) #endif am_wal_redo_postgres = true; + /* + * Pageserver treats any output to stderr as an ERROR, so we must + * set the log level as early as possible to only log FATAL and + * above during WAL redo (note that loglevel ERROR also logs LOG, + * which is super strange but that's not something we can solve + * for here. ¯\_(-_-)_/¯ + */ + SetConfigOption("log_min_messages", "FATAL", PGC_SUSET, PGC_S_OVERRIDE); + SetConfigOption("client_min_messages", "ERROR", PGC_SUSET, + PGC_S_OVERRIDE); /* * WAL redo does not need a large number of buffers. And speed of diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 0c0c06aa86..0e9f823860 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1631,7 +1631,6 @@ class NeonPageserver(PgProtocol): ".*took more than expected to complete.*", # these can happen during shutdown, but it should not be a reason to fail a test ".*completed, took longer than expected.*", - '.*registered custom resource manager \\\\"neon\\\\".*', # AWS S3 may emit 500 errors for keys in a DeleteObjects response: we retry these # and it is not a failure of our code when it happens. ".*DeleteObjects.*We encountered an internal error. Please try again.*", From 8360307ea0acfa4069533eadbd284a2abf1d80e1 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 26 Oct 2023 14:00:26 +0100 Subject: [PATCH 26/69] pageserver: exponential backoff on compaction/GC failures (#5672) Previously, if walredo process crashed we would try to spawn a fresh one every 2 seconds, which is expensive in itself, but also results in a high I/O load from the part of the compaction prior to the failure, which we re-run every 2 seconds. Closes: https://github.com/neondatabase/neon/issues/5671 --- pageserver/src/tenant/tasks.rs | 40 ++++++++++++++++++++++----- test_runner/fixtures/neon_fixtures.py | 6 ++-- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/pageserver/src/tenant/tasks.rs b/pageserver/src/tenant/tasks.rs index a845456181..f419d2e73a 100644 --- a/pageserver/src/tenant/tasks.rs +++ b/pageserver/src/tenant/tasks.rs @@ -12,7 +12,7 @@ use crate::task_mgr::{TaskKind, BACKGROUND_RUNTIME}; use crate::tenant::{Tenant, TenantState}; use tokio_util::sync::CancellationToken; use tracing::*; -use utils::completion; +use utils::{backoff, completion}; static CONCURRENT_BACKGROUND_TASKS: once_cell::sync::Lazy = once_cell::sync::Lazy::new(|| { @@ -139,7 +139,10 @@ pub fn start_background_loops( /// Compaction task's main loop /// async fn compaction_loop(tenant: Arc, cancel: CancellationToken) { - let wait_duration = Duration::from_secs(2); + const MAX_BACKOFF_SECS: f64 = 300.0; + // How many errors we have seen consequtively + let mut error_run_count = 0; + TENANT_TASK_EVENTS.with_label_values(&["start"]).inc(); async { let ctx = RequestContext::todo_child(TaskKind::Compaction, DownloadBehavior::Download); @@ -173,9 +176,19 @@ async fn compaction_loop(tenant: Arc, cancel: CancellationToken) { } else { // Run compaction if let Err(e) = tenant.compaction_iteration(&cancel, &ctx).await { - error!("Compaction failed, retrying in {:?}: {e:?}", wait_duration); - wait_duration + let wait_duration = backoff::exponential_backoff_duration_seconds( + error_run_count, + 1.0, + MAX_BACKOFF_SECS, + ); + error_run_count += 1; + error!( + "Compaction failed {error_run_count} times, retrying in {:?}: {e:?}", + wait_duration + ); + Duration::from_secs_f64(wait_duration) } else { + error_run_count = 0; period } }; @@ -210,7 +223,10 @@ async fn compaction_loop(tenant: Arc, cancel: CancellationToken) { /// GC task's main loop /// async fn gc_loop(tenant: Arc, cancel: CancellationToken) { - let wait_duration = Duration::from_secs(2); + const MAX_BACKOFF_SECS: f64 = 300.0; + // How many errors we have seen consequtively + let mut error_run_count = 0; + TENANT_TASK_EVENTS.with_label_values(&["start"]).inc(); async { // GC might require downloading, to find the cutoff LSN that corresponds to the @@ -249,9 +265,19 @@ async fn gc_loop(tenant: Arc, cancel: CancellationToken) { .gc_iteration(None, gc_horizon, tenant.get_pitr_interval(), &ctx) .await; if let Err(e) = res { - error!("Gc failed, retrying in {:?}: {e:?}", wait_duration); - wait_duration + let wait_duration = backoff::exponential_backoff_duration_seconds( + error_run_count, + 1.0, + MAX_BACKOFF_SECS, + ); + error_run_count += 1; + error!( + "Gc failed {error_run_count} times, retrying in {:?}: {e:?}", + wait_duration + ); + Duration::from_secs_f64(wait_duration) } else { + error_run_count = 0; period } }; diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 0e9f823860..9184f0c43a 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1616,16 +1616,16 @@ class NeonPageserver(PgProtocol): ".*wait for layer upload ops to complete.*", # .*Caused by:.*wait_completion aborted because upload queue was stopped ".*gc_loop.*Gc failed, retrying in.*timeline is Stopping", # When gc checks timeline state after acquiring layer_removal_cs ".*gc_loop.*Gc failed, retrying in.*: Cannot run GC iteration on inactive tenant", # Tenant::gc precondition - ".*compaction_loop.*Compaction failed, retrying in.*timeline or pageserver is shutting down", # When compaction checks timeline state after acquiring layer_removal_cs + ".*compaction_loop.*Compaction failed.*, retrying in.*timeline or pageserver is shutting down", # When compaction checks timeline state after acquiring layer_removal_cs ".*query handler for 'pagestream.*failed: Timeline .* was not found", # postgres reconnects while timeline_delete doesn't hold the tenant's timelines.lock() ".*query handler for 'pagestream.*failed: Timeline .* is not active", # timeline delete in progress ".*task iteration took longer than the configured period.*", # this is until #3501 - ".*Compaction failed, retrying in [^:]+: Cannot run compaction iteration on inactive tenant", + ".*Compaction failed.*, retrying in [^:]+: Cannot run compaction iteration on inactive tenant", # these can happen anytime we do compactions from background task and shutdown pageserver r".*ERROR.*ancestor timeline \S+ is being stopped", # this is expected given our collaborative shutdown approach for the UploadQueue - ".*Compaction failed, retrying in .*: queue is in state Stopped.*", + ".*Compaction failed.*, retrying in .*: queue is in state Stopped.*", # Pageserver timeline deletion should be polled until it gets 404, so ignore it globally ".*Error processing HTTP request: NotFound: Timeline .* was not found", ".*took more than expected to complete.*", From de90bf4663396546673812d20aa1650c702f57a2 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 26 Oct 2023 14:48:44 +0100 Subject: [PATCH 27/69] pageserver: always load remote metadata (no more `spawn_load`) (#5580) ## Problem The pageserver had two ways of loading a tenant: - `spawn_load` would trust on-disk content to reflect all existing timelines - `spawn_attach` would list timelines in remote storage. It was incorrect for `spawn_load` to trust local disk content, because it doesn't know if the tenant might have been attached and written somewhere else. To make this correct would requires some generation number checks, but the payoff is to avoid one S3 op per tenant at startup, so it's not worth the complexity -- it is much simpler to have one way to load a tenant. ## Summary of changes - `Tenant` objects are always created with `Tenant::spawn`: there is no more distinction between "load" and "attach". - The ability to run without remote storage (for `neon_local`) is preserved by adding a branch inside `attach` that uses a fallback `load_local` if no remote_storage is present. - Fix attaching a tenant when it has a timeline with no IndexPart: this can occur if a newly created timeline manages to upload a layer before it has uploaded an index. - The attach marker file that used to indicate whether a tenant should be "loaded" or "attached" is no longer needed, and is removed. - The GenericRemoteStorage interface gets a `list()` method that maps more directly to what ListObjects does, returning both keys and common prefixes. The existing `list_files` and `list_prefixes` methods are just calls into `list()` now -- these can be removed later if we would like to shrink the interface a bit. - The remote deletion marker is moved into `timelines/` and detected as part of listing timelines rather than as a separate GET request. If any existing tenants have a marker in the old location (unlikely, only happens if something crashes mid-delete), then they will rely on the control plane retrying to complete their deletion. - Revise S3 calls for timeline listing and tenant load to take a cancellation token, and retry forever: it never makes sense to make a Tenant broken because of a transient S3 issue. ## Breaking changes - The remote deletion marker is moved from `deleted` to `timelines/deleted` within the tenant prefix. Markers in the old location will be ignored: it is the control plane's responsibility to retry deletions until they succeed. Markers in the new location will be tolerated by the previous release of pageserver via https://github.com/neondatabase/neon/pull/5632 - The local `attaching` marker file is no longer written. Therefore, if the pageserver is downgraded after running this code, the old pageserver will not be able to distinguish between partially attached tenants and fully attached tenants. This would only impact tenants that were partway through attaching at the moment of downgrade. In the unlikely even t that we do experience an incident that prompts us to roll back, then we may check for attach operations in flight, and manually insert `attaching` marker files as needed. --------- Co-authored-by: Christian Schwarz --- libs/pageserver_api/src/models.rs | 1 - libs/remote_storage/src/azure_blob.rs | 61 +- libs/remote_storage/src/lib.rs | 53 +- libs/remote_storage/src/local_fs.rs | 155 +++- libs/remote_storage/src/s3_bucket.rs | 103 +-- libs/remote_storage/src/simulate_failures.rs | 13 +- pageserver/src/config.rs | 8 +- pageserver/src/tenant.rs | 847 +++++++----------- pageserver/src/tenant/delete.rs | 98 +- pageserver/src/tenant/mgr.rs | 134 ++- .../src/tenant/remote_timeline_client.rs | 42 +- .../tenant/remote_timeline_client/download.rs | 106 ++- pageserver/src/tenant/timeline/delete.rs | 1 + test_runner/fixtures/neon_fixtures.py | 35 + test_runner/regress/test_branching.py | 23 +- test_runner/regress/test_broken_timeline.py | 22 +- .../regress/test_pageserver_restart.py | 6 +- test_runner/regress/test_tenant_delete.py | 3 +- test_runner/regress/test_tenant_detach.py | 50 +- test_runner/regress/test_tenants.py | 48 +- .../regress/test_threshold_based_eviction.py | 3 +- test_runner/regress/test_timeline_delete.py | 7 - 22 files changed, 808 insertions(+), 1011 deletions(-) diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index b2064cb7fe..e667645c0a 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -110,7 +110,6 @@ impl TenantState { // So, return `Maybe` while Attaching, making Console wait for the attach task to finish. Self::Attaching | Self::Activating(ActivatingFrom::Attaching) => Maybe, // tenant mgr startup distinguishes attaching from loading via marker file. - // If it's loading, there is no attach marker file, i.e., attach had finished in the past. Self::Loading | Self::Activating(ActivatingFrom::Loading) => Attached, // We only reach Active after successful load / attach. // So, call atttachment status Attached. diff --git a/libs/remote_storage/src/azure_blob.rs b/libs/remote_storage/src/azure_blob.rs index 152929ecd3..f0521d27c5 100644 --- a/libs/remote_storage/src/azure_blob.rs +++ b/libs/remote_storage/src/azure_blob.rs @@ -23,8 +23,8 @@ use tracing::debug; use crate::s3_bucket::RequestKind; use crate::{ - AzureConfig, ConcurrencyLimiter, Download, DownloadError, RemotePath, RemoteStorage, - StorageMetadata, + AzureConfig, ConcurrencyLimiter, Download, DownloadError, Listing, ListingMode, RemotePath, + RemoteStorage, StorageMetadata, }; pub struct AzureBlobStorage { @@ -184,10 +184,11 @@ fn to_download_error(error: azure_core::Error) -> DownloadError { #[async_trait::async_trait] impl RemoteStorage for AzureBlobStorage { - async fn list_prefixes( + async fn list( &self, prefix: Option<&RemotePath>, - ) -> Result, DownloadError> { + mode: ListingMode, + ) -> anyhow::Result { // get the passed prefix or if it is not set use prefix_in_bucket value let list_prefix = prefix .map(|p| self.relative_path_to_name(p)) @@ -195,16 +196,19 @@ impl RemoteStorage for AzureBlobStorage { .map(|mut p| { // required to end with a separator // otherwise request will return only the entry of a prefix - if !p.ends_with(REMOTE_STORAGE_PREFIX_SEPARATOR) { + if matches!(mode, ListingMode::WithDelimiter) + && !p.ends_with(REMOTE_STORAGE_PREFIX_SEPARATOR) + { p.push(REMOTE_STORAGE_PREFIX_SEPARATOR); } p }); - let mut builder = self - .client - .list_blobs() - .delimiter(REMOTE_STORAGE_PREFIX_SEPARATOR.to_string()); + let mut builder = self.client.list_blobs(); + + if let ListingMode::WithDelimiter = mode { + builder = builder.delimiter(REMOTE_STORAGE_PREFIX_SEPARATOR.to_string()); + } if let Some(prefix) = list_prefix { builder = builder.prefix(Cow::from(prefix.to_owned())); @@ -215,46 +219,23 @@ impl RemoteStorage for AzureBlobStorage { } let mut response = builder.into_stream(); - let mut res = Vec::new(); - while let Some(entry) = response.next().await { - let entry = entry.map_err(to_download_error)?; - let name_iter = entry + let mut res = Listing::default(); + while let Some(l) = response.next().await { + let entry = l.map_err(to_download_error)?; + let prefix_iter = entry .blobs .prefixes() .map(|prefix| self.name_to_relative_path(&prefix.name)); - res.extend(name_iter); - } - Ok(res) - } + res.prefixes.extend(prefix_iter); - async fn list_files(&self, folder: Option<&RemotePath>) -> anyhow::Result> { - let folder_name = folder - .map(|p| self.relative_path_to_name(p)) - .or_else(|| self.prefix_in_container.clone()); - - let mut builder = self.client.list_blobs(); - - if let Some(folder_name) = folder_name { - builder = builder.prefix(Cow::from(folder_name.to_owned())); - } - - if let Some(limit) = self.max_keys_per_list_response { - builder = builder.max_results(MaxResults::new(limit)); - } - - let mut response = builder.into_stream(); - let mut res = Vec::new(); - while let Some(l) = response.next().await { - let entry = l.map_err(anyhow::Error::new)?; - let name_iter = entry + let blob_iter = entry .blobs .blobs() - .map(|bl| self.name_to_relative_path(&bl.name)); - res.extend(name_iter); + .map(|k| self.name_to_relative_path(&k.name)); + res.keys.extend(blob_iter); } Ok(res) } - async fn upload( &self, mut from: impl AsyncRead + Unpin + Send + Sync + 'static, diff --git a/libs/remote_storage/src/lib.rs b/libs/remote_storage/src/lib.rs index 435364d83a..9b5b340db0 100644 --- a/libs/remote_storage/src/lib.rs +++ b/libs/remote_storage/src/lib.rs @@ -129,6 +129,22 @@ impl RemotePath { } } +/// We don't need callers to be able to pass arbitrary delimiters: just control +/// whether listings will use a '/' separator or not. +/// +/// The WithDelimiter mode will populate `prefixes` and `keys` in the result. The +/// NoDelimiter mode will only populate `keys`. +pub enum ListingMode { + WithDelimiter, + NoDelimiter, +} + +#[derive(Default)] +pub struct Listing { + pub prefixes: Vec, + pub keys: Vec, +} + /// Storage (potentially remote) API to manage its state. /// This storage tries to be unaware of any layered repository context, /// providing basic CRUD operations for storage files. @@ -141,8 +157,13 @@ pub trait RemoteStorage: Send + Sync + 'static { async fn list_prefixes( &self, prefix: Option<&RemotePath>, - ) -> Result, DownloadError>; - + ) -> Result, DownloadError> { + let result = self + .list(prefix, ListingMode::WithDelimiter) + .await? + .prefixes; + Ok(result) + } /// Lists all files in directory "recursively" /// (not really recursively, because AWS has a flat namespace) /// Note: This is subtely different than list_prefixes, @@ -154,7 +175,16 @@ pub trait RemoteStorage: Send + Sync + 'static { /// whereas, /// list_prefixes("foo/bar/") = ["cat", "dog"] /// See `test_real_s3.rs` for more details. - async fn list_files(&self, folder: Option<&RemotePath>) -> anyhow::Result>; + async fn list_files(&self, prefix: Option<&RemotePath>) -> anyhow::Result> { + let result = self.list(prefix, ListingMode::NoDelimiter).await?.keys; + Ok(result) + } + + async fn list( + &self, + prefix: Option<&RemotePath>, + _mode: ListingMode, + ) -> anyhow::Result; /// Streams the local file contents into remote into the remote storage entry. async fn upload( @@ -205,6 +235,9 @@ pub enum DownloadError { BadInput(anyhow::Error), /// The file was not found in the remote storage. NotFound, + /// A cancellation token aborted the download, typically during + /// tenant detach or process shutdown. + Cancelled, /// The file was found in the remote storage, but the download failed. Other(anyhow::Error), } @@ -215,6 +248,7 @@ impl std::fmt::Display for DownloadError { DownloadError::BadInput(e) => { write!(f, "Failed to download a remote file due to user input: {e}") } + DownloadError::Cancelled => write!(f, "Cancelled, shutting down"), DownloadError::NotFound => write!(f, "No file found for the remote object id given"), DownloadError::Other(e) => write!(f, "Failed to download a remote file: {e:?}"), } @@ -234,6 +268,19 @@ pub enum GenericRemoteStorage { } impl GenericRemoteStorage { + pub async fn list( + &self, + prefix: Option<&RemotePath>, + mode: ListingMode, + ) -> anyhow::Result { + match self { + Self::LocalFs(s) => s.list(prefix, mode).await, + Self::AwsS3(s) => s.list(prefix, mode).await, + Self::AzureBlob(s) => s.list(prefix, mode).await, + Self::Unreliable(s) => s.list(prefix, mode).await, + } + } + // A function for listing all the files in a "directory" // Example: // list_files("foo/bar") = ["foo/bar/a.txt", "foo/bar/b.txt"] diff --git a/libs/remote_storage/src/local_fs.rs b/libs/remote_storage/src/local_fs.rs index 3d32b6b631..1be50ce565 100644 --- a/libs/remote_storage/src/local_fs.rs +++ b/libs/remote_storage/src/local_fs.rs @@ -15,7 +15,7 @@ use tokio::{ use tracing::*; use utils::{crashsafe::path_with_suffix_extension, fs_ext::is_directory_empty}; -use crate::{Download, DownloadError, RemotePath}; +use crate::{Download, DownloadError, Listing, ListingMode, RemotePath}; use super::{RemoteStorage, StorageMetadata}; @@ -75,7 +75,7 @@ impl LocalFs { } #[cfg(test)] - async fn list(&self) -> anyhow::Result> { + async fn list_all(&self) -> anyhow::Result> { Ok(get_all_files(&self.storage_root, true) .await? .into_iter() @@ -89,52 +89,10 @@ impl LocalFs { }) .collect()) } -} - -#[async_trait::async_trait] -impl RemoteStorage for LocalFs { - async fn list_prefixes( - &self, - prefix: Option<&RemotePath>, - ) -> Result, DownloadError> { - let path = match prefix { - Some(prefix) => Cow::Owned(prefix.with_base(&self.storage_root)), - None => Cow::Borrowed(&self.storage_root), - }; - - let prefixes_to_filter = get_all_files(path.as_ref(), false) - .await - .map_err(DownloadError::Other)?; - - let mut prefixes = Vec::with_capacity(prefixes_to_filter.len()); - - // filter out empty directories to mirror s3 behavior. - for prefix in prefixes_to_filter { - if prefix.is_dir() - && is_directory_empty(&prefix) - .await - .map_err(DownloadError::Other)? - { - continue; - } - - prefixes.push( - prefix - .strip_prefix(&self.storage_root) - .context("Failed to strip prefix") - .and_then(RemotePath::new) - .expect( - "We list files for storage root, hence should be able to remote the prefix", - ), - ) - } - - Ok(prefixes) - } // recursively lists all files in a directory, // mirroring the `list_files` for `s3_bucket` - async fn list_files(&self, folder: Option<&RemotePath>) -> anyhow::Result> { + async fn list_recursive(&self, folder: Option<&RemotePath>) -> anyhow::Result> { let full_path = match folder { Some(folder) => folder.with_base(&self.storage_root), None => self.storage_root.clone(), @@ -186,6 +144,70 @@ impl RemoteStorage for LocalFs { Ok(files) } +} + +#[async_trait::async_trait] +impl RemoteStorage for LocalFs { + async fn list( + &self, + prefix: Option<&RemotePath>, + mode: ListingMode, + ) -> Result { + let mut result = Listing::default(); + + if let ListingMode::NoDelimiter = mode { + let keys = self + .list_recursive(prefix) + .await + .map_err(DownloadError::Other)?; + + result.keys = keys + .into_iter() + .filter(|k| { + let path = k.with_base(&self.storage_root); + !path.is_dir() + }) + .collect(); + + return Ok(result); + } + + let path = match prefix { + Some(prefix) => Cow::Owned(prefix.with_base(&self.storage_root)), + None => Cow::Borrowed(&self.storage_root), + }; + + let prefixes_to_filter = get_all_files(path.as_ref(), false) + .await + .map_err(DownloadError::Other)?; + + // filter out empty directories to mirror s3 behavior. + for prefix in prefixes_to_filter { + if prefix.is_dir() + && is_directory_empty(&prefix) + .await + .map_err(DownloadError::Other)? + { + continue; + } + + let stripped = prefix + .strip_prefix(&self.storage_root) + .context("Failed to strip prefix") + .and_then(RemotePath::new) + .expect( + "We list files for storage root, hence should be able to remote the prefix", + ); + + if prefix.is_dir() { + result.prefixes.push(stripped); + } else { + result.keys.push(stripped); + } + } + + Ok(result) + } async fn upload( &self, @@ -479,7 +501,7 @@ mod fs_tests { let target_path_1 = upload_dummy_file(&storage, "upload_1", None).await?; assert_eq!( - storage.list().await?, + storage.list_all().await?, vec![target_path_1.clone()], "Should list a single file after first upload" ); @@ -667,7 +689,7 @@ mod fs_tests { let upload_target = upload_dummy_file(&storage, upload_name, None).await?; storage.delete(&upload_target).await?; - assert!(storage.list().await?.is_empty()); + assert!(storage.list_all().await?.is_empty()); storage .delete(&upload_target) @@ -725,6 +747,43 @@ mod fs_tests { Ok(()) } + #[tokio::test] + async fn list() -> anyhow::Result<()> { + // No delimiter: should recursively list everything + let storage = create_storage()?; + let child = upload_dummy_file(&storage, "grandparent/parent/child", None).await?; + let uncle = upload_dummy_file(&storage, "grandparent/uncle", None).await?; + + let listing = storage.list(None, ListingMode::NoDelimiter).await?; + assert!(listing.prefixes.is_empty()); + assert_eq!(listing.keys, [uncle.clone(), child.clone()].to_vec()); + + // Delimiter: should only go one deep + let listing = storage.list(None, ListingMode::WithDelimiter).await?; + + assert_eq!( + listing.prefixes, + [RemotePath::from_string("timelines").unwrap()].to_vec() + ); + assert!(listing.keys.is_empty()); + + // Delimiter & prefix + let listing = storage + .list( + Some(&RemotePath::from_string("timelines/some_timeline/grandparent").unwrap()), + ListingMode::WithDelimiter, + ) + .await?; + assert_eq!( + listing.prefixes, + [RemotePath::from_string("timelines/some_timeline/grandparent/parent").unwrap()] + .to_vec() + ); + assert_eq!(listing.keys, [uncle.clone()].to_vec()); + + Ok(()) + } + async fn upload_dummy_file( storage: &LocalFs, name: &str, @@ -777,7 +836,7 @@ mod fs_tests { } async fn list_files_sorted(storage: &LocalFs) -> anyhow::Result> { - let mut files = storage.list().await?; + let mut files = storage.list_all().await?; files.sort_by(|a, b| a.0.cmp(&b.0)); Ok(files) } diff --git a/libs/remote_storage/src/s3_bucket.rs b/libs/remote_storage/src/s3_bucket.rs index fc94281666..560a2c14e9 100644 --- a/libs/remote_storage/src/s3_bucket.rs +++ b/libs/remote_storage/src/s3_bucket.rs @@ -30,8 +30,8 @@ use tracing::debug; use super::StorageMetadata; use crate::{ - ConcurrencyLimiter, Download, DownloadError, RemotePath, RemoteStorage, S3Config, - MAX_KEYS_PER_DELETE, REMOTE_STORAGE_PREFIX_SEPARATOR, + ConcurrencyLimiter, Download, DownloadError, Listing, ListingMode, RemotePath, RemoteStorage, + S3Config, MAX_KEYS_PER_DELETE, REMOTE_STORAGE_PREFIX_SEPARATOR, }; pub(super) mod metrics; @@ -299,13 +299,13 @@ impl AsyncRead for TimedDownload { #[async_trait::async_trait] impl RemoteStorage for S3Bucket { - /// See the doc for `RemoteStorage::list_prefixes` - /// Note: it wont include empty "directories" - async fn list_prefixes( + async fn list( &self, prefix: Option<&RemotePath>, - ) -> Result, DownloadError> { + mode: ListingMode, + ) -> Result { let kind = RequestKind::List; + let mut result = Listing::default(); // get the passed prefix or if it is not set use prefix_in_bucket value let list_prefix = prefix @@ -314,28 +314,33 @@ impl RemoteStorage for S3Bucket { .map(|mut p| { // required to end with a separator // otherwise request will return only the entry of a prefix - if !p.ends_with(REMOTE_STORAGE_PREFIX_SEPARATOR) { + if matches!(mode, ListingMode::WithDelimiter) + && !p.ends_with(REMOTE_STORAGE_PREFIX_SEPARATOR) + { p.push(REMOTE_STORAGE_PREFIX_SEPARATOR); } p }); - let mut document_keys = Vec::new(); - let mut continuation_token = None; loop { let _guard = self.permit(kind).await; let started_at = start_measuring_requests(kind); - let fetch_response = self + let mut request = self .client .list_objects_v2() .bucket(self.bucket_name.clone()) .set_prefix(list_prefix.clone()) .set_continuation_token(continuation_token) - .delimiter(REMOTE_STORAGE_PREFIX_SEPARATOR.to_string()) - .set_max_keys(self.max_keys_per_list_response) + .set_max_keys(self.max_keys_per_list_response); + + if let ListingMode::WithDelimiter = mode { + request = request.delimiter(REMOTE_STORAGE_PREFIX_SEPARATOR.to_string()); + } + + let response = request .send() .await .context("Failed to list S3 prefixes") @@ -345,71 +350,35 @@ impl RemoteStorage for S3Bucket { metrics::BUCKET_METRICS .req_seconds - .observe_elapsed(kind, &fetch_response, started_at); + .observe_elapsed(kind, &response, started_at); - let fetch_response = fetch_response?; + let response = response?; - document_keys.extend( - fetch_response - .common_prefixes - .unwrap_or_default() - .into_iter() + let keys = response.contents().unwrap_or_default(); + let empty = Vec::new(); + let prefixes = response.common_prefixes.as_ref().unwrap_or(&empty); + + tracing::info!("list: {} prefixes, {} keys", prefixes.len(), keys.len()); + + for object in keys { + let object_path = object.key().expect("response does not contain a key"); + let remote_path = self.s3_object_to_relative_path(object_path); + result.keys.push(remote_path); + } + + result.prefixes.extend( + prefixes + .iter() .filter_map(|o| Some(self.s3_object_to_relative_path(o.prefix()?))), ); - continuation_token = match fetch_response.next_continuation_token { + continuation_token = match response.next_continuation_token { Some(new_token) => Some(new_token), None => break, }; } - Ok(document_keys) - } - - /// See the doc for `RemoteStorage::list_files` - async fn list_files(&self, folder: Option<&RemotePath>) -> anyhow::Result> { - let kind = RequestKind::List; - - let folder_name = folder - .map(|p| self.relative_path_to_s3_object(p)) - .or_else(|| self.prefix_in_bucket.clone()); - - // AWS may need to break the response into several parts - let mut continuation_token = None; - let mut all_files = vec![]; - loop { - let _guard = self.permit(kind).await; - let started_at = start_measuring_requests(kind); - - let response = self - .client - .list_objects_v2() - .bucket(self.bucket_name.clone()) - .set_prefix(folder_name.clone()) - .set_continuation_token(continuation_token) - .set_max_keys(self.max_keys_per_list_response) - .send() - .await - .context("Failed to list files in S3 bucket"); - - let started_at = ScopeGuard::into_inner(started_at); - metrics::BUCKET_METRICS - .req_seconds - .observe_elapsed(kind, &response, started_at); - - let response = response?; - - for object in response.contents().unwrap_or_default() { - let object_path = object.key().expect("response does not contain a key"); - let remote_path = self.s3_object_to_relative_path(object_path); - all_files.push(remote_path); - } - match response.next_continuation_token { - Some(new_token) => continuation_token = Some(new_token), - None => break, - } - } - Ok(all_files) + Ok(result) } async fn upload( diff --git a/libs/remote_storage/src/simulate_failures.rs b/libs/remote_storage/src/simulate_failures.rs index 6d6a5c1d24..cd13db1923 100644 --- a/libs/remote_storage/src/simulate_failures.rs +++ b/libs/remote_storage/src/simulate_failures.rs @@ -5,7 +5,9 @@ use std::collections::hash_map::Entry; use std::collections::HashMap; use std::sync::Mutex; -use crate::{Download, DownloadError, RemotePath, RemoteStorage, StorageMetadata}; +use crate::{ + Download, DownloadError, Listing, ListingMode, RemotePath, RemoteStorage, StorageMetadata, +}; pub struct UnreliableWrapper { inner: crate::GenericRemoteStorage, @@ -95,6 +97,15 @@ impl RemoteStorage for UnreliableWrapper { self.inner.list_files(folder).await } + async fn list( + &self, + prefix: Option<&RemotePath>, + mode: ListingMode, + ) -> Result { + self.attempt(RemoteOp::ListPrefixes(prefix.cloned()))?; + self.inner.list(prefix, mode).await + } + async fn upload( &self, data: impl tokio::io::AsyncRead + Unpin + Send + Sync + 'static, diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index fe62a7299a..a58c08e29b 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -33,8 +33,7 @@ use crate::disk_usage_eviction_task::DiskUsageEvictionTaskConfig; use crate::tenant::config::TenantConf; use crate::tenant::config::TenantConfOpt; use crate::tenant::{ - TENANTS_SEGMENT_NAME, TENANT_ATTACHING_MARKER_FILENAME, TENANT_DELETED_MARKER_FILE_NAME, - TIMELINES_SEGMENT_NAME, + TENANTS_SEGMENT_NAME, TENANT_DELETED_MARKER_FILE_NAME, TIMELINES_SEGMENT_NAME, }; use crate::{ IGNORED_TENANT_FILE_NAME, METADATA_FILE_NAME, TENANT_CONFIG_NAME, TENANT_LOCATION_CONFIG_NAME, @@ -633,11 +632,6 @@ impl PageServerConf { self.tenants_path().join(tenant_id.to_string()) } - pub fn tenant_attaching_mark_file_path(&self, tenant_id: &TenantId) -> Utf8PathBuf { - self.tenant_path(tenant_id) - .join(TENANT_ATTACHING_MARKER_FILENAME) - } - pub fn tenant_ignore_mark_file_path(&self, tenant_id: &TenantId) -> Utf8PathBuf { self.tenant_path(tenant_id).join(IGNORED_TENANT_FILE_NAME) } diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 919291a7e7..3a426ac87b 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -24,8 +24,8 @@ use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; use tracing::*; use utils::completion; -use utils::completion::Completion; use utils::crashsafe::path_with_suffix_extension; +use utils::fs_ext; use std::cmp::min; use std::collections::hash_map::Entry; @@ -78,7 +78,6 @@ use crate::tenant::remote_timeline_client::MaybeDeletedIndexPart; use crate::tenant::storage_layer::DeltaLayer; use crate::tenant::storage_layer::ImageLayer; use crate::InitializationOrder; -use crate::METADATA_FILE_NAME; use crate::tenant::timeline::delete::DeleteTimelineFlow; use crate::tenant::timeline::uninit::cleanup_timeline_directory; @@ -152,8 +151,6 @@ pub const TENANTS_SEGMENT_NAME: &str = "tenants"; /// Parts of the `.neon/tenants//timelines/` directory prefix. pub const TIMELINES_SEGMENT_NAME: &str = "timelines"; -pub const TENANT_ATTACHING_MARKER_FILENAME: &str = "attaching"; - pub const TENANT_DELETED_MARKER_FILE_NAME: &str = "deleted"; /// References to shared objects that are passed into each tenant, such @@ -192,6 +189,18 @@ struct TimelinePreload { index_part: Result, } +pub(crate) struct TenantPreload { + deleting: bool, + timelines: HashMap, +} + +/// When we spawn a tenant, there is a special mode for tenant creation that +/// avoids trying to read anything from remote storage. +pub(crate) enum SpawnMode { + Normal, + Create, +} + /// /// Tenant consists of multiple timelines. Keep them in a hash table. /// @@ -390,12 +399,6 @@ pub enum CreateTimelineError { Other(#[from] anyhow::Error), } -/// spawn_attach argument for whether the caller is using attachment markers -pub(super) enum AttachMarkerMode { - Expect, - Ignore, -} - struct TenantDirectoryScan { sorted_timelines_to_load: Vec<(TimelineId, TimelineMetadata)>, timelines_to_resume_deletion: Vec<(TimelineId, Option)>, @@ -408,7 +411,6 @@ enum CreateTimelineCause { impl Tenant { /// Yet another helper for timeline initialization. - /// Contains the common part of `load_local_timeline` and `load_remote_timeline`. /// /// - Initializes the Timeline struct and inserts it into the tenant's hash map /// - Scans the local timeline directory for layer files and builds the layer map @@ -508,7 +510,6 @@ impl Tenant { Ok(()) } - /// /// Attach a tenant that's available in cloud storage. /// /// This returns quickly, after just creating the in-memory object @@ -518,16 +519,17 @@ impl Tenant { /// finishes. You can use wait_until_active() to wait for the task to /// complete. /// - pub(crate) fn spawn_attach( + #[allow(clippy::too_many_arguments)] + pub(crate) fn spawn( conf: &'static PageServerConf, tenant_id: TenantId, resources: TenantSharedResources, attached_conf: AttachedTenantConf, + init_order: Option, tenants: &'static tokio::sync::RwLock, - expect_marker: AttachMarkerMode, + mode: SpawnMode, ctx: &RequestContext, ) -> anyhow::Result> { - // TODO dedup with spawn_load let wal_redo_manager = Arc::new(WalRedoManager::from(PostgresRedoManager::new( conf, tenant_id, ))); @@ -561,22 +563,61 @@ impl Tenant { false, async move { // Ideally we should use Tenant::set_broken_no_wait, but it is not supposed to be used when tenant is in loading state. - let make_broken = |t: &Tenant, err: anyhow::Error| { - error!("attach failed, setting tenant state to Broken: {err:?}"); - t.state.send_modify(|state| { - assert_eq!( - *state, - TenantState::Attaching, + let make_broken = + |t: &Tenant, err: anyhow::Error| { + error!("attach failed, setting tenant state to Broken: {err:?}"); + t.state.send_modify(|state| { + // The Stopping case is for when we have passed control on to DeleteTenantFlow: + // if it errors, we will call make_broken when tenant is already in Stopping. + assert!( + matches!(*state, TenantState::Attaching | TenantState::Stopping { .. }), "the attach task owns the tenant state until activation is complete" ); - *state = TenantState::broken_from_reason(err.to_string()); - }); + + *state = TenantState::broken_from_reason(err.to_string()); + }); + }; + + let mut init_order = init_order; + // take the completion because initial tenant loading will complete when all of + // these tasks complete. + let _completion = init_order + .as_mut() + .and_then(|x| x.initial_tenant_load.take()); + let remote_load_completion = init_order + .as_mut() + .and_then(|x| x.initial_tenant_load_remote.take()); + + let preload = match mode { + SpawnMode::Create => {None}, + SpawnMode::Normal => { + match &remote_storage { + Some(remote_storage) => Some( + match tenant_clone + .preload(remote_storage, task_mgr::shutdown_token()) + .instrument( + tracing::info_span!(parent: None, "attach_preload", tenant_id=%tenant_id), + ) + .await { + Ok(p) => p, + Err(e) => { + make_broken(&tenant_clone, anyhow::anyhow!(e)); + return Ok(()); + } + }, + ), + None => None, + } + } }; + // Remote preload is complete. + drop(remote_load_completion); + let pending_deletion = { match DeleteTenantFlow::should_resume_deletion( conf, - remote_storage.as_ref(), + preload.as_ref().map(|p| p.deleting).unwrap_or(false), &tenant_clone, ) .await @@ -592,10 +633,27 @@ impl Tenant { info!("pending_deletion {}", pending_deletion.is_some()); if let Some(deletion) = pending_deletion { + // as we are no longer loading, signal completion by dropping + // the completion while we resume deletion + drop(_completion); + // do not hold to initial_logical_size_attempt as it will prevent loading from proceeding without timeout + let _ = init_order + .as_mut() + .and_then(|x| x.initial_logical_size_attempt.take()); + let background_jobs_can_start = + init_order.as_ref().map(|x| &x.background_jobs_can_start); + if let Some(background) = background_jobs_can_start { + info!("waiting for backgound jobs barrier"); + background.clone().wait().await; + info!("ready for backgound jobs barrier"); + } + match DeleteTenantFlow::resume_from_attach( deletion, &tenant_clone, + preload, tenants, + init_order, &ctx, ) .await @@ -608,7 +666,7 @@ impl Tenant { } } - match tenant_clone.attach(&ctx, expect_marker).await { + match tenant_clone.attach(init_order, preload, &ctx).await { Ok(()) => { info!("attach finished, activating"); tenant_clone.activate(broker_client, None, &ctx); @@ -628,6 +686,42 @@ impl Tenant { Ok(tenant) } + pub(crate) async fn preload( + self: &Arc, + remote_storage: &GenericRemoteStorage, + cancel: CancellationToken, + ) -> anyhow::Result { + // Get list of remote timelines + // download index files for every tenant timeline + info!("listing remote timelines"); + let (remote_timeline_ids, other_keys) = remote_timeline_client::list_remote_timelines( + remote_storage, + self.tenant_id, + cancel.clone(), + ) + .await?; + + let deleting = other_keys.contains(TENANT_DELETED_MARKER_FILE_NAME); + info!( + "found {} timelines, deleting={}", + remote_timeline_ids.len(), + deleting + ); + + for k in other_keys { + if k != TENANT_DELETED_MARKER_FILE_NAME { + warn!("Unexpected non timeline key {k}"); + } + } + + Ok(TenantPreload { + deleting, + timelines: self + .load_timeline_metadata(remote_timeline_ids, remote_storage, cancel) + .await?, + }) + } + /// /// Background task that downloads all data for a tenant and brings it to Active state. /// @@ -635,89 +729,62 @@ impl Tenant { /// async fn attach( self: &Arc, + mut init_order: Option, + preload: Option, ctx: &RequestContext, - expect_marker: AttachMarkerMode, ) -> anyhow::Result<()> { span::debug_assert_current_span_has_tenant_id(); - let marker_file = self.conf.tenant_attaching_mark_file_path(&self.tenant_id); - if let AttachMarkerMode::Expect = expect_marker { - if !tokio::fs::try_exists(&marker_file) - .await - .context("check for existence of marker file")? - { - anyhow::bail!( - "implementation error: marker file should exist at beginning of this function" - ); + crate::failpoint_support::sleep_millis_async!("before-attaching-tenant"); + + let preload = match preload { + Some(p) => p, + None => { + // Deprecated dev mode: load from local disk state instead of remote storage + // https://github.com/neondatabase/neon/issues/5624 + return self.load_local(init_order, ctx).await; } - } + }; - // Get list of remote timelines - // download index files for every tenant timeline - info!("listing remote timelines"); - - let remote_storage = self - .remote_storage - .as_ref() - .ok_or_else(|| anyhow::anyhow!("cannot attach without remote storage"))?; - - let remote_timeline_ids = - remote_timeline_client::list_remote_timelines(remote_storage, self.tenant_id).await?; - - info!("found {} timelines", remote_timeline_ids.len()); - - // Download & parse index parts - let mut part_downloads = JoinSet::new(); - for timeline_id in remote_timeline_ids { - let client = RemoteTimelineClient::new( - remote_storage.clone(), - self.deletion_queue_client.clone(), - self.conf, - self.tenant_id, - timeline_id, - self.generation, - ); - part_downloads.spawn( - async move { - debug!("starting index part download"); - - let index_part = client - .download_index_file() - .await - .context("download index file")?; - - debug!("finished index part download"); - - Result::<_, anyhow::Error>::Ok((timeline_id, client, index_part)) - } - .map(move |res| { - res.with_context(|| format!("download index part for timeline {timeline_id}")) - }) - .instrument(info_span!("download_index_part", %timeline_id)), - ); - } + // Signal that we have completed remote phase + init_order + .as_mut() + .and_then(|x| x.initial_tenant_load_remote.take()); let mut timelines_to_resume_deletions = vec![]; - // Wait for all the download tasks to complete & collect results. let mut remote_index_and_client = HashMap::new(); let mut timeline_ancestors = HashMap::new(); - while let Some(result) = part_downloads.join_next().await { - // NB: we already added timeline_id as context to the error - let result: Result<_, anyhow::Error> = result.context("joinset task join")?; - let (timeline_id, client, index_part) = result?; - debug!("successfully downloaded index part for timeline {timeline_id}"); + let mut existent_timelines = HashSet::new(); + for (timeline_id, preload) in preload.timelines { + // In this context a timeline "exists" if it has any content in remote storage: this will + // be our cue to not delete any corresponding local directory + existent_timelines.insert(timeline_id); + + let index_part = match preload.index_part { + Ok(i) => { + debug!("remote index part exists for timeline {timeline_id}"); + i + } + Err(e) => { + // Timeline creation is not atomic: we might upload a layer but no index_part. We expect + // that the creation will be retried by the control plane and eventually result in + // a valid loadable state. + warn!(%timeline_id, "Failed to load index_part from remote storage, failed creation? ({e})"); + continue; + } + }; match index_part { MaybeDeletedIndexPart::IndexPart(index_part) => { timeline_ancestors.insert(timeline_id, index_part.metadata.clone()); - remote_index_and_client.insert(timeline_id, (index_part, client)); + remote_index_and_client.insert(timeline_id, (index_part, preload.client)); } MaybeDeletedIndexPart::Deleted(index_part) => { info!( "timeline {} is deleted, picking to resume deletion", timeline_id ); - timelines_to_resume_deletions.push((timeline_id, index_part, client)); + timelines_to_resume_deletions.push((timeline_id, index_part, preload.client)); } } } @@ -771,12 +838,9 @@ impl Tenant { .map_err(LoadLocalTimelineError::ResumeDeletion)?; } - if let AttachMarkerMode::Expect = expect_marker { - std::fs::remove_file(&marker_file) - .with_context(|| format!("unlink attach marker file {marker_file}"))?; - crashsafe::fsync(marker_file.parent().expect("marker file has parent dir")) - .context("fsync tenant directory after unlinking attach marker file")?; - } + // The local filesystem contents are a cache of what's in the remote IndexPart; + // IndexPart is the source of truth. + self.clean_up_timelines(&existent_timelines)?; crate::failpoint_support::sleep_millis_async!("attach-before-activate"); @@ -785,6 +849,69 @@ impl Tenant { Ok(()) } + /// Check for any local timeline directories that are temporary, or do not correspond to a + /// timeline that still exists: this can happen if we crashed during a deletion/creation, or + /// if a timeline was deleted while the tenant was attached to a different pageserver. + fn clean_up_timelines(&self, existent_timelines: &HashSet) -> anyhow::Result<()> { + let timelines_dir = self.conf.timelines_path(&self.tenant_id); + + let entries = match timelines_dir.read_dir_utf8() { + Ok(d) => d, + Err(e) => { + if e.kind() == std::io::ErrorKind::NotFound { + return Ok(()); + } else { + return Err(e).context("list timelines directory for tenant"); + } + } + }; + + for entry in entries { + let entry = entry.context("read timeline dir entry")?; + let entry_path = entry.path(); + + let purge = if crate::is_temporary(entry_path) + // TODO: uninit_mark isn't needed any more, since uninitialized timelines are already + // covered by the check that the timeline must exist in remote storage. + || is_uninit_mark(entry_path) + || crate::is_delete_mark(entry_path) + { + true + } else { + match TimelineId::try_from(entry_path.file_name()) { + Ok(i) => { + // Purge if the timeline ID does not exist in remote storage: remote storage is the authority. + !existent_timelines.contains(&i) + } + Err(e) => { + tracing::warn!( + "Unparseable directory in timelines directory: {entry_path}, ignoring ({e})" + ); + // Do not purge junk: if we don't recognize it, be cautious and leave it for a human. + false + } + } + }; + + if purge { + tracing::info!("Purging stale timeline dentry {entry_path}"); + if let Err(e) = match entry.file_type() { + Ok(t) => if t.is_dir() { + std::fs::remove_dir_all(entry_path) + } else { + std::fs::remove_file(entry_path) + } + .or_else(fs_ext::ignore_not_found), + Err(e) => Err(e), + } { + tracing::warn!("Failed to purge stale timeline dentry {entry_path}: {e}"); + } + } + } + + Ok(()) + } + /// Get sum of all remote timelines sizes /// /// This function relies on the index_part instead of listing the remote storage @@ -874,152 +1001,6 @@ impl Tenant { )) } - /// Load a tenant that's available on local disk - /// - /// This is used at pageserver startup, to rebuild the in-memory - /// structures from on-disk state. This is similar to attaching a tenant, - /// but the index files already exist on local disk, as well as some layer - /// files. - /// - /// If the loading fails for some reason, the Tenant will go into Broken - /// state. - #[instrument(skip_all, fields(tenant_id=%tenant_id))] - pub(crate) fn spawn_load( - conf: &'static PageServerConf, - tenant_id: TenantId, - attached_conf: AttachedTenantConf, - resources: TenantSharedResources, - init_order: Option, - tenants: &'static tokio::sync::RwLock, - ctx: &RequestContext, - ) -> Arc { - span::debug_assert_current_span_has_tenant_id(); - - let broker_client = resources.broker_client; - let remote_storage = resources.remote_storage; - - let wal_redo_manager = Arc::new(WalRedoManager::from(PostgresRedoManager::new( - conf, tenant_id, - ))); - let tenant = Tenant::new( - TenantState::Loading, - conf, - attached_conf, - wal_redo_manager, - tenant_id, - remote_storage.clone(), - resources.deletion_queue_client.clone(), - ); - let tenant = Arc::new(tenant); - - // Do all the hard work in a background task - let tenant_clone = Arc::clone(&tenant); - - let ctx = ctx.detached_child(TaskKind::InitialLoad, DownloadBehavior::Warn); - let _ = task_mgr::spawn( - &tokio::runtime::Handle::current(), - TaskKind::InitialLoad, - Some(tenant_id), - None, - "initial tenant load", - false, - async move { - // Ideally we should use Tenant::set_broken_no_wait, but it is not supposed to be used when tenant is in loading state. - let make_broken = |t: &Tenant, err: anyhow::Error| { - error!("load failed, setting tenant state to Broken: {err:?}"); - t.state.send_modify(|state| { - assert!( - matches!(*state, TenantState::Loading | TenantState::Stopping { .. }), - "the loading task owns the tenant state until activation is complete" - ); - *state = TenantState::broken_from_reason(err.to_string()); - }); - }; - - let mut init_order = init_order; - - // take the completion because initial tenant loading will complete when all of - // these tasks complete. - let _completion = init_order - .as_mut() - .and_then(|x| x.initial_tenant_load.take()); - let remote_load_completion = init_order - .as_mut() - .and_then(|x| x.initial_tenant_load_remote.take()); - - // Dont block pageserver startup on figuring out deletion status - let pending_deletion = { - match DeleteTenantFlow::should_resume_deletion( - conf, - remote_storage.as_ref(), - &tenant_clone, - ) - .await - { - Ok(should_resume_deletion) => should_resume_deletion, - Err(err) => { - make_broken(&tenant_clone, anyhow::anyhow!(err)); - return Ok(()); - } - } - }; - - info!("pending deletion {}", pending_deletion.is_some()); - - if let Some(deletion) = pending_deletion { - // as we are no longer loading, signal completion by dropping - // the completion while we resume deletion - drop(_completion); - drop(remote_load_completion); - // do not hold to initial_logical_size_attempt as it will prevent loading from proceeding without timeout - let _ = init_order - .as_mut() - .and_then(|x| x.initial_logical_size_attempt.take()); - - match DeleteTenantFlow::resume_from_load( - deletion, - &tenant_clone, - init_order.as_ref(), - tenants, - &ctx, - ) - .await - { - Err(err) => { - make_broken(&tenant_clone, anyhow::anyhow!(err)); - return Ok(()); - } - Ok(()) => return Ok(()), - } - } - - let background_jobs_can_start = - init_order.as_ref().map(|x| &x.background_jobs_can_start); - - match tenant_clone - .load(init_order.as_ref(), remote_load_completion, &ctx) - .await - { - Ok(()) => { - debug!("load finished"); - - tenant_clone.activate(broker_client, background_jobs_can_start, &ctx); - } - Err(err) => make_broken(&tenant_clone, err), - } - - Ok(()) - } - .instrument({ - let span = tracing::info_span!(parent: None, "load", tenant_id=%tenant_id); - span.follows_from(Span::current()); - span - }), - ); - - tenant - } - fn scan_and_sort_timelines_dir(self: Arc) -> anyhow::Result { let mut timelines_to_load: HashMap = HashMap::new(); // Note timelines_to_resume_deletion needs to be separate because it can be not sortable @@ -1167,6 +1148,7 @@ impl Tenant { self: &Arc, timeline_ids: HashSet, remote_storage: &GenericRemoteStorage, + cancel: CancellationToken, ) -> anyhow::Result> { let mut part_downloads = JoinSet::new(); for timeline_id in timeline_ids { @@ -1178,11 +1160,12 @@ impl Tenant { timeline_id, self.generation, ); + let cancel_clone = cancel.clone(); part_downloads.spawn( async move { debug!("starting index part download"); - let index_part = client.download_index_file().await; + let index_part = client.download_index_file(cancel_clone).await; debug!("finished index part download"); @@ -1200,10 +1183,25 @@ impl Tenant { } let mut timeline_preloads: HashMap = HashMap::new(); - while let Some(result) = part_downloads.join_next().await { - let preload_result = result.context("join preload task")?; - let preload = preload_result?; - timeline_preloads.insert(preload.timeline_id, preload); + + loop { + tokio::select!( + next = part_downloads.join_next() => { + match next { + Some(result) => { + let preload_result = result.context("join preload task")?; + let preload = preload_result?; + timeline_preloads.insert(preload.timeline_id, preload); + }, + None => { + break; + } + } + }, + _ = cancel.cancelled() => { + anyhow::bail!("Cancelled while waiting for remote index download") + } + ) } Ok(timeline_preloads) @@ -1214,10 +1212,9 @@ impl Tenant { /// files on disk. Used at pageserver startup. /// /// No background tasks are started as part of this routine. - async fn load( + async fn load_local( self: &Arc, - init_order: Option<&InitializationOrder>, - remote_completion: Option, + init_order: Option, ctx: &RequestContext, ) -> anyhow::Result<()> { span::debug_assert_current_span_has_tenant_id(); @@ -1242,38 +1239,10 @@ impl Tenant { // FIXME original collect_timeline_files contained one more check: // 1. "Timeline has no ancestor and no layer files" - // Load remote content for timelines in this tenant - let all_timeline_ids = scan - .sorted_timelines_to_load - .iter() - .map(|i| i.0) - .chain(scan.timelines_to_resume_deletion.iter().map(|i| i.0)) - .collect(); - let mut preload = if let Some(remote_storage) = &self.remote_storage { - Some( - self.load_timeline_metadata(all_timeline_ids, remote_storage) - .await?, - ) - } else { - None - }; - - drop(remote_completion); - - crate::failpoint_support::sleep_millis_async!("before-loading-tenant"); - // Process loadable timelines first for (timeline_id, local_metadata) in scan.sorted_timelines_to_load { - let timeline_preload = preload.as_mut().map(|p| p.remove(&timeline_id).unwrap()); if let Err(e) = self - .load_local_timeline( - timeline_id, - local_metadata, - timeline_preload, - init_order, - ctx, - false, - ) + .load_local_timeline(timeline_id, local_metadata, init_order.as_ref(), ctx, false) .await { match e { @@ -1306,14 +1275,11 @@ impl Tenant { } } Some(local_metadata) => { - let timeline_preload = - preload.as_mut().map(|p| p.remove(&timeline_id).unwrap()); if let Err(e) = self .load_local_timeline( timeline_id, local_metadata, - timeline_preload, - init_order, + init_order.as_ref(), ctx, true, ) @@ -1344,183 +1310,36 @@ impl Tenant { /// Subroutine of `load_tenant`, to load an individual timeline /// /// NB: The parent is assumed to be already loaded! - #[instrument(skip(self, local_metadata, init_order, preload, ctx))] + #[instrument(skip(self, local_metadata, init_order, ctx))] async fn load_local_timeline( self: &Arc, timeline_id: TimelineId, local_metadata: TimelineMetadata, - preload: Option, init_order: Option<&InitializationOrder>, ctx: &RequestContext, found_delete_mark: bool, ) -> Result<(), LoadLocalTimelineError> { span::debug_assert_current_span_has_tenant_id(); - let mut resources = self.build_timeline_resources(timeline_id); + let resources = self.build_timeline_resources(timeline_id); - struct RemoteStartupData { - index_part: IndexPart, - remote_metadata: TimelineMetadata, + if found_delete_mark { + // There is no remote client, we found local metadata. + // Continue cleaning up local disk. + DeleteTimelineFlow::resume_deletion( + Arc::clone(self), + timeline_id, + &local_metadata, + None, + self.deletion_queue_client.clone(), + init_order, + ) + .await + .context("resume deletion") + .map_err(LoadLocalTimelineError::ResumeDeletion)?; + return Ok(()); } - let (remote_startup_data, remote_client) = match preload { - Some(preload) => { - let TimelinePreload { - index_part, - client: remote_client, - timeline_id: _timeline_id, - } = preload; - match index_part { - Ok(index_part) => { - let index_part = match index_part { - MaybeDeletedIndexPart::IndexPart(index_part) => index_part, - MaybeDeletedIndexPart::Deleted(index_part) => { - // TODO: we won't reach here if remote storage gets de-configured after start of the deletion operation. - // Example: - // start deletion operation - // finishes upload of index part - // pageserver crashes - // remote storage gets de-configured - // pageserver starts - // - // We don't really anticipate remote storage to be de-configured, so, for now, this is fine. - // Also, maybe we'll remove that option entirely in the future, see https://github.com/neondatabase/neon/issues/4099. - info!("is_deleted is set on remote, resuming removal of timeline data originally done by timeline deletion handler"); - - remote_client - .init_upload_queue_stopped_to_continue_deletion(&index_part) - .context("init queue stopped") - .map_err(LoadLocalTimelineError::ResumeDeletion)?; - - DeleteTimelineFlow::resume_deletion( - Arc::clone(self), - timeline_id, - &local_metadata, - Some(remote_client), - self.deletion_queue_client.clone(), - init_order, - ) - .await - .context("resume deletion") - .map_err(LoadLocalTimelineError::ResumeDeletion)?; - - return Ok(()); - } - }; - - let remote_metadata = index_part.metadata.clone(); - ( - Some(RemoteStartupData { - index_part, - remote_metadata, - }), - Some(remote_client), - ) - } - Err(DownloadError::NotFound) => { - info!(found_delete_mark, "no index file was found on the remote, resuming deletion or cleaning unuploaded up"); - - if found_delete_mark { - // We could've resumed at a point where remote index was deleted, but metadata file wasnt. - // Cleanup: - return DeleteTimelineFlow::cleanup_remaining_timeline_fs_traces( - self, - timeline_id, - ) - .await - .context("cleanup_remaining_timeline_fs_traces") - .map_err(LoadLocalTimelineError::ResumeDeletion); - } - - // as the remote index_part.json did not exist, this timeline is a - // not-yet-uploaded one. it should be deleted now, because the branching might - // not have been valid as it's ancestor may have been restored to earlier state - // as well. in practice, control plane will keep retrying. - // - // first ensure that the un-uploaded timeline looks like it should, as in we - // are not accidentially deleting a timeline which was ever active: - // - root timelines have metadata and one possibly partial layer - // - branched timelines have metadata - // - // if the timeline does not look like expected, fail loading of the tenant. - // cleaning the timeline up manually and reloading the tenant is possible via - // the above log message. - let path = self.conf.timeline_path(&self.tenant_id, &timeline_id); - - let span = tracing::Span::current(); - - return tokio::task::spawn_blocking({ - move || { - use std::str::FromStr; - use crate::tenant::storage_layer::LayerFileName; - - let _e = span.entered(); - let mut metadata = false; - let mut layers = 0; - let mut others = 0; - for dentry in path.read_dir_utf8()? { - let dentry = dentry?; - let file_name = dentry.file_name(); - - if file_name == METADATA_FILE_NAME { - metadata = true; - continue; - } - - if LayerFileName::from_str(file_name).is_ok() - { - layers += 1; - continue; - } - - others += 1; - } - - // bootstrapped have the one image layer file, or one partial temp - // file, branched have just the metadata - if !(metadata && layers + others <= 1) { - anyhow::bail!("unexpected assumed unuploaded, never been active timeline: found metadata={}, layers={}, others={}", metadata, layers, others); - } - - let tmp_path = - path.with_file_name(format!("{timeline_id}{}", TEMP_FILE_SUFFIX)); - std::fs::rename(path, &tmp_path)?; - std::fs::remove_dir_all(&tmp_path)?; - Ok(()) - } - }) - .await - .map_err(anyhow::Error::new) - .and_then(|x| x) - .context("delete assumed unuploaded fresh timeline") - .map_err(LoadLocalTimelineError::Load); - } - Err(e) => return Err(LoadLocalTimelineError::Load(anyhow::Error::new(e))), - } - } - None => { - if found_delete_mark { - // There is no remote client, we found local metadata. - // Continue cleaning up local disk. - DeleteTimelineFlow::resume_deletion( - Arc::clone(self), - timeline_id, - &local_metadata, - None, - self.deletion_queue_client.clone(), - init_order, - ) - .await - .context("resume deletion") - .map_err(LoadLocalTimelineError::ResumeDeletion)?; - return Ok(()); - } - - (None, resources.remote_client) - } - }; - resources.remote_client = remote_client; - let ancestor = if let Some(ancestor_timeline_id) = local_metadata.ancestor_timeline() { let ancestor_timeline = self.get_timeline(ancestor_timeline_id, false) .with_context(|| anyhow::anyhow!("cannot find ancestor timeline {ancestor_timeline_id} for timeline {timeline_id}")) @@ -1530,27 +1349,11 @@ impl Tenant { None }; - let (index_part, metadata) = match remote_startup_data { - Some(RemoteStartupData { - index_part, - remote_metadata, - }) => { - // always choose the remote metadata to be crash consistent (see RFC 27) - save_metadata(self.conf, &self.tenant_id, &timeline_id, &remote_metadata) - .await - .context("save_metadata") - .map_err(LoadLocalTimelineError::Load)?; - - (Some(index_part), remote_metadata) - } - None => (None, local_metadata), - }; - self.timeline_init_and_sync( timeline_id, resources, - index_part, - metadata, + None, + local_metadata, ancestor, init_order, ctx, @@ -3387,16 +3190,10 @@ fn remove_timeline_and_uninit_mark( Ok(()) } -pub(crate) enum CreateTenantFilesMode { - Create, - Attach, -} - pub(crate) async fn create_tenant_files( conf: &'static PageServerConf, location_conf: &LocationConf, tenant_id: &TenantId, - mode: CreateTenantFilesMode, ) -> anyhow::Result { let target_tenant_directory = conf.tenant_path(tenant_id); anyhow::ensure!( @@ -3419,7 +3216,6 @@ pub(crate) async fn create_tenant_files( conf, location_conf, tenant_id, - mode, &temporary_tenant_dir, &target_tenant_directory, ) @@ -3445,28 +3241,9 @@ async fn try_create_target_tenant_dir( conf: &'static PageServerConf, location_conf: &LocationConf, tenant_id: &TenantId, - mode: CreateTenantFilesMode, temporary_tenant_dir: &Utf8Path, target_tenant_directory: &Utf8Path, ) -> Result<(), anyhow::Error> { - match mode { - CreateTenantFilesMode::Create => {} // needs no attach marker, writing tenant conf + atomic rename of dir is good enough - CreateTenantFilesMode::Attach => { - let attach_marker_path = temporary_tenant_dir.join(TENANT_ATTACHING_MARKER_FILENAME); - let file = std::fs::OpenOptions::new() - .create_new(true) - .write(true) - .open(&attach_marker_path) - .with_context(|| { - format!("could not create attach marker file {attach_marker_path:?}") - })?; - file.sync_all().with_context(|| { - format!("could not sync attach marker file: {attach_marker_path:?}") - })?; - // fsync of the directory in which the file resides comes later in this function - } - } - let temporary_tenant_timelines_dir = rebase_directory( &conf.timelines_path(tenant_id), target_tenant_directory, @@ -3684,6 +3461,11 @@ pub(crate) mod harness { } } + enum LoadMode { + Local, + Remote, + } + pub struct TenantHarness { pub conf: &'static PageServerConf, pub tenant_conf: TenantConf, @@ -3767,7 +3549,36 @@ pub(crate) mod harness { ) } - pub async fn try_load(&self, ctx: &RequestContext) -> anyhow::Result> { + fn remote_empty(&self) -> bool { + let tenant_path = self.conf.tenant_path(&self.tenant_id); + let remote_tenant_dir = self + .remote_fs_dir + .join(tenant_path.strip_prefix(&self.conf.workdir).unwrap()); + if std::fs::metadata(&remote_tenant_dir).is_err() { + return true; + } + + match std::fs::read_dir(remote_tenant_dir) + .unwrap() + .flatten() + .next() + { + Some(entry) => { + tracing::debug!( + "remote_empty: not empty, found file {}", + entry.file_name().to_string_lossy(), + ); + false + } + None => true, + } + } + + async fn do_try_load( + &self, + ctx: &RequestContext, + mode: LoadMode, + ) -> anyhow::Result> { let walredo_mgr = Arc::new(WalRedoManager::from(TestRedoManager)); let tenant = Arc::new(Tenant::new( @@ -3783,12 +3594,26 @@ pub(crate) mod harness { Some(self.remote_storage.clone()), self.deletion_queue.new_client(), )); - tenant - .load(None, None, ctx) - .instrument(info_span!("try_load", tenant_id=%self.tenant_id)) - .await?; - // TODO reuse Tenant::activate (needs broker) + match mode { + LoadMode::Local => { + tenant + .load_local(None, ctx) + .instrument(info_span!("try_load", tenant_id=%self.tenant_id)) + .await?; + } + LoadMode::Remote => { + let preload = tenant + .preload(&self.remote_storage, CancellationToken::new()) + .instrument(info_span!("try_load_preload", tenant_id=%self.tenant_id)) + .await?; + tenant + .attach(None, Some(preload), ctx) + .instrument(info_span!("try_load", tenant_id=%self.tenant_id)) + .await?; + } + } + tenant.state.send_replace(TenantState::Active); for timeline in tenant.timelines.lock().unwrap().values() { timeline.set_state(TimelineState::Active); @@ -3796,6 +3621,27 @@ pub(crate) mod harness { Ok(tenant) } + /// For tests that specifically want to exercise the local load path, which does + /// not use remote storage. + pub async fn try_load_local(&self, ctx: &RequestContext) -> anyhow::Result> { + self.do_try_load(ctx, LoadMode::Local).await + } + + /// The 'load' in this function is either a local load or a normal attachment, + pub async fn try_load(&self, ctx: &RequestContext) -> anyhow::Result> { + // If we have nothing in remote storage, must use load_local instead of attach: attach + // will error out if there are no timelines. + // + // See https://github.com/neondatabase/neon/issues/5456 for how we will eliminate + // this weird state of a Tenant which exists but doesn't have any timelines. + let mode = match self.remote_empty() { + true => LoadMode::Local, + false => LoadMode::Remote, + }; + + self.do_try_load(ctx, mode).await + } + pub fn timeline_path(&self, timeline_id: &TimelineId) -> Utf8PathBuf { self.conf.timeline_path(&self.tenant_id, timeline_id) } @@ -4364,7 +4210,7 @@ mod tests { } #[tokio::test] - async fn corrupt_metadata() -> anyhow::Result<()> { + async fn corrupt_local_metadata() -> anyhow::Result<()> { const TEST_NAME: &str = "corrupt_metadata"; let harness = TenantHarness::create(TEST_NAME)?; let (tenant, ctx) = harness.load().await; @@ -4382,16 +4228,19 @@ mod tests { .unwrap(); drop(tenant); + // Corrupt local metadata let metadata_path = harness.timeline_path(&TIMELINE_ID).join(METADATA_FILE_NAME); - assert!(metadata_path.is_file()); - let mut metadata_bytes = std::fs::read(&metadata_path)?; assert_eq!(metadata_bytes.len(), 512); metadata_bytes[8] ^= 1; std::fs::write(metadata_path, metadata_bytes)?; - let err = harness.try_load(&ctx).await.err().expect("should fail"); + let err = harness + .try_load_local(&ctx) + .await + .err() + .expect("should fail"); // get all the stack with all .context, not only the last one let message = format!("{err:#}"); let expected = "failed to load metadata"; diff --git a/pageserver/src/tenant/delete.rs b/pageserver/src/tenant/delete.rs index 989f2cd779..3e2d6ca7bb 100644 --- a/pageserver/src/tenant/delete.rs +++ b/pageserver/src/tenant/delete.rs @@ -3,10 +3,10 @@ use std::sync::Arc; use anyhow::Context; use camino::{Utf8Path, Utf8PathBuf}; use pageserver_api::models::TenantState; -use remote_storage::{DownloadError, GenericRemoteStorage, RemotePath}; +use remote_storage::{GenericRemoteStorage, RemotePath}; use tokio::sync::OwnedMutexGuard; use tokio_util::sync::CancellationToken; -use tracing::{error, info, instrument, warn, Instrument, Span}; +use tracing::{error, instrument, warn, Instrument, Span}; use utils::{ backoff, completion, crashsafe, fs_ext, @@ -25,11 +25,9 @@ use super::{ remote_timeline_client::{FAILED_REMOTE_OP_RETRIES, FAILED_UPLOAD_WARN_THRESHOLD}, span, timeline::delete::DeleteTimelineFlow, - tree_sort_timelines, DeleteTimelineError, Tenant, + tree_sort_timelines, DeleteTimelineError, Tenant, TenantPreload, }; -const SHOULD_RESUME_DELETION_FETCH_MARK_ATTEMPTS: u32 = 3; - #[derive(Debug, thiserror::Error)] pub(crate) enum DeleteTenantError { #[error("GetTenant {0}")] @@ -60,7 +58,7 @@ fn remote_tenant_delete_mark_path( .context("Failed to strip workdir prefix") .and_then(RemotePath::new) .context("tenant path")?; - Ok(tenant_remote_path.join(Utf8Path::new("deleted"))) + Ok(tenant_remote_path.join(Utf8Path::new("timelines/deleted"))) } async fn create_remote_delete_mark( @@ -239,32 +237,6 @@ async fn cleanup_remaining_fs_traces( Ok(()) } -pub(crate) async fn remote_delete_mark_exists( - conf: &PageServerConf, - tenant_id: &TenantId, - remote_storage: &GenericRemoteStorage, -) -> anyhow::Result { - // If remote storage is there we rely on it - let remote_mark_path = remote_tenant_delete_mark_path(conf, tenant_id).context("path")?; - - let result = backoff::retry( - || async { remote_storage.download(&remote_mark_path).await }, - |e| matches!(e, DownloadError::NotFound), - SHOULD_RESUME_DELETION_FETCH_MARK_ATTEMPTS, - SHOULD_RESUME_DELETION_FETCH_MARK_ATTEMPTS, - "fetch_tenant_deletion_mark", - // TODO: use a cancellation token (https://github.com/neondatabase/neon/issues/5066) - backoff::Cancel::new(CancellationToken::new(), || unreachable!()), - ) - .await; - - match result { - Ok(_) => Ok(true), - Err(DownloadError::NotFound) => Ok(false), - Err(e) => Err(anyhow::anyhow!(e)).context("remote_delete_mark_exists")?, - } -} - /// Orchestrates tenant shut down of all tasks, removes its in-memory structures, /// and deletes its data from both disk and s3. /// The sequence of steps: @@ -276,10 +248,9 @@ pub(crate) async fn remote_delete_mark_exists( /// 6. Remove remote mark /// 7. Cleanup remaining fs traces, tenant dir, config, timelines dir, local delete mark /// It is resumable from any step in case a crash/restart occurs. -/// There are three entrypoints to the process: +/// There are two entrypoints to the process: /// 1. [`DeleteTenantFlow::run`] this is the main one called by a management api handler. -/// 2. [`DeleteTenantFlow::resume_from_load`] is called during restarts when local or remote deletion marks are still there. -/// 3. [`DeleteTenantFlow::resume_from_attach`] is called when deletion is resumed tenant is found to be deleted during attach process. +/// 2. [`DeleteTenantFlow::resume_from_attach`] is called when deletion is resumed tenant is found to be deleted during attach process. /// Note the only other place that messes around timeline delete mark is the `Tenant::spawn_load` function. #[derive(Default)] pub enum DeleteTenantFlow { @@ -378,7 +349,7 @@ impl DeleteTenantFlow { pub(crate) async fn should_resume_deletion( conf: &'static PageServerConf, - remote_storage: Option<&GenericRemoteStorage>, + remote_mark_exists: bool, tenant: &Tenant, ) -> Result, DeleteTenantError> { let acquire = |t: &Tenant| { @@ -389,66 +360,25 @@ impl DeleteTenantFlow { ) }; - let tenant_id = tenant.tenant_id; - // Check local mark first, if its there there is no need to go to s3 to check whether remote one exists. - if conf.tenant_deleted_mark_file_path(&tenant_id).exists() { + if remote_mark_exists { return Ok(acquire(tenant)); } - let remote_storage = match remote_storage { - Some(remote_storage) => remote_storage, - None => return Ok(None), - }; - - if remote_delete_mark_exists(conf, &tenant_id, remote_storage).await? { + let tenant_id = tenant.tenant_id; + // Check local mark first, if its there there is no need to go to s3 to check whether remote one exists. + if conf.tenant_deleted_mark_file_path(&tenant_id).exists() { Ok(acquire(tenant)) } else { Ok(None) } } - pub(crate) async fn resume_from_load( - guard: DeletionGuard, - tenant: &Arc, - init_order: Option<&InitializationOrder>, - tenants: &'static tokio::sync::RwLock, - ctx: &RequestContext, - ) -> Result<(), DeleteTenantError> { - let (_, progress) = completion::channel(); - - tenant - .set_stopping(progress, true, false) - .await - .expect("cant be stopping or broken"); - - // Do not consume valuable resources during the load phase, continue deletion once init phase is complete. - let background_jobs_can_start = init_order.as_ref().map(|x| &x.background_jobs_can_start); - if let Some(background) = background_jobs_can_start { - info!("waiting for backgound jobs barrier"); - background.clone().wait().await; - info!("ready for backgound jobs barrier"); - } - - // Tenant may not be loadable if we fail late in cleanup_remaining_fs_traces (e g remove timelines dir) - let timelines_path = tenant.conf.timelines_path(&tenant.tenant_id); - if timelines_path.exists() { - tenant.load(init_order, None, ctx).await.context("load")?; - } - - Self::background( - guard, - tenant.conf, - tenant.remote_storage.clone(), - tenants, - tenant, - ) - .await - } - pub(crate) async fn resume_from_attach( guard: DeletionGuard, tenant: &Arc, + preload: Option, tenants: &'static tokio::sync::RwLock, + init_order: Option, ctx: &RequestContext, ) -> Result<(), DeleteTenantError> { let (_, progress) = completion::channel(); @@ -459,7 +389,7 @@ impl DeleteTenantFlow { .expect("cant be stopping or broken"); tenant - .attach(ctx, super::AttachMarkerMode::Expect) + .attach(init_order, preload, ctx) .await .context("attach")?; diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 380b610a4c..33fdc76f8d 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -26,10 +26,7 @@ use crate::deletion_queue::DeletionQueueClient; use crate::task_mgr::{self, TaskKind}; use crate::tenant::config::{AttachmentMode, LocationConf, LocationMode, TenantConfOpt}; use crate::tenant::delete::DeleteTenantFlow; -use crate::tenant::{ - create_tenant_files, AttachMarkerMode, AttachedTenantConf, CreateTenantFilesMode, Tenant, - TenantState, -}; +use crate::tenant::{create_tenant_files, AttachedTenantConf, SpawnMode, Tenant, TenantState}; use crate::{InitializationOrder, IGNORED_TENANT_FILE_NAME, TEMP_FILE_SUFFIX}; use utils::crashsafe::path_with_suffix_extension; @@ -437,14 +434,15 @@ pub async fn init_tenant_mgr( location_conf.attach_in_generation(generation); Tenant::persist_tenant_config(conf, &tenant_id, &location_conf).await?; - match schedule_local_tenant_processing( + match tenant_spawn( conf, tenant_id, &tenant_dir_path, - AttachedTenantConf::try_from(location_conf)?, resources.clone(), + AttachedTenantConf::try_from(location_conf)?, Some(init_order.clone()), &TENANTS, + SpawnMode::Normal, &ctx, ) { Ok(tenant) => { @@ -464,15 +462,18 @@ pub async fn init_tenant_mgr( Ok(()) } +/// Wrapper for Tenant::spawn that checks invariants before running, and inserts +/// a broken tenant in the map if Tenant::spawn fails. #[allow(clippy::too_many_arguments)] -pub(crate) fn schedule_local_tenant_processing( +pub(crate) fn tenant_spawn( conf: &'static PageServerConf, tenant_id: TenantId, tenant_path: &Utf8Path, - location_conf: AttachedTenantConf, resources: TenantSharedResources, + location_conf: AttachedTenantConf, init_order: Option, tenants: &'static tokio::sync::RwLock, + mode: SpawnMode, ctx: &RequestContext, ) -> anyhow::Result> { anyhow::ensure!( @@ -496,45 +497,24 @@ pub(crate) fn schedule_local_tenant_processing( "Cannot load tenant, ignore mark found at {tenant_ignore_mark:?}" ); - let tenant = if conf.tenant_attaching_mark_file_path(&tenant_id).exists() { - info!("tenant {tenant_id} has attaching mark file, resuming its attach operation"); - if resources.remote_storage.is_none() { - warn!("tenant {tenant_id} has attaching mark file, but pageserver has no remote storage configured"); - Tenant::create_broken_tenant( - conf, - tenant_id, - "attaching mark file present but no remote storage configured".to_string(), - ) - } else { - match Tenant::spawn_attach( - conf, - tenant_id, - resources, - location_conf, - tenants, - AttachMarkerMode::Expect, - ctx, - ) { - Ok(tenant) => tenant, - Err(e) => { - error!("Failed to spawn_attach tenant {tenant_id}, reason: {e:#}"); - Tenant::create_broken_tenant(conf, tenant_id, format!("{e:#}")) - } - } + info!("Attaching tenant {tenant_id}"); + let tenant = match Tenant::spawn( + conf, + tenant_id, + resources, + location_conf, + init_order, + tenants, + mode, + ctx, + ) { + Ok(tenant) => tenant, + Err(e) => { + error!("Failed to spawn tenant {tenant_id}, reason: {e:#}"); + Tenant::create_broken_tenant(conf, tenant_id, format!("{e:#}")) } - } else { - info!("tenant {tenant_id} is assumed to be loadable, starting load operation"); - // Start loading the tenant into memory. It will initially be in Loading state. - Tenant::spawn_load( - conf, - tenant_id, - location_conf, - resources, - init_order, - tenants, - ctx, - ) }; + Ok(tenant) } @@ -670,29 +650,41 @@ pub(crate) async fn create_tenant( ctx: &RequestContext, ) -> Result, TenantMapInsertError> { tenant_map_insert(tenant_id, || async { - let location_conf = LocationConf::attached_single(tenant_conf, generation); // We're holding the tenants lock in write mode while doing local IO. // If this section ever becomes contentious, introduce a new `TenantState::Creating` // and do the work in that state. - let tenant_directory = super::create_tenant_files(conf, &location_conf, &tenant_id, CreateTenantFilesMode::Create).await?; + super::create_tenant_files(conf, &location_conf, &tenant_id).await?; + // TODO: tenant directory remains on disk if we bail out from here on. // See https://github.com/neondatabase/neon/issues/4233 - let created_tenant = - schedule_local_tenant_processing(conf, tenant_id, &tenant_directory, - AttachedTenantConf::try_from(location_conf)?, resources, None, &TENANTS, ctx)?; + let tenant_path = conf.tenant_path(&tenant_id); + + let created_tenant = tenant_spawn( + conf, + tenant_id, + &tenant_path, + resources, + AttachedTenantConf::try_from(location_conf)?, + None, + &TENANTS, + SpawnMode::Create, + ctx, + )?; // TODO: tenant object & its background loops remain, untracked in tenant map, if we fail here. // See https://github.com/neondatabase/neon/issues/4233 let crated_tenant_id = created_tenant.tenant_id(); anyhow::ensure!( - tenant_id == crated_tenant_id, - "loaded created tenant has unexpected tenant id (expect {tenant_id} != actual {crated_tenant_id})", - ); + tenant_id == crated_tenant_id, + "loaded created tenant has unexpected tenant id \ + (expect {tenant_id} != actual {crated_tenant_id})", + ); Ok(created_tenant) - }).await + }) + .await } #[derive(Debug, thiserror::Error)] @@ -801,9 +793,10 @@ pub(crate) async fn upsert_location( } } + let tenant_path = conf.tenant_path(&tenant_id); + let new_slot = match &new_location_config.mode { LocationMode::Secondary(_) => { - let tenant_path = conf.tenant_path(&tenant_id); // Directory doesn't need to be fsync'd because if we crash it can // safely be recreated next time this tenant location is configured. unsafe_create_dir_all(&tenant_path) @@ -833,28 +826,21 @@ pub(crate) async fn upsert_location( .await .map_err(SetNewTenantConfigError::Persist)?; - let tenant = match Tenant::spawn_attach( + let tenant = tenant_spawn( conf, tenant_id, + &tenant_path, TenantSharedResources { broker_client, remote_storage, deletion_queue_client, }, AttachedTenantConf::try_from(new_location_config)?, + None, &TENANTS, - // The LocationConf API does not use marker files, because we have Secondary - // locations where the directory's existence is not a signal that it contains - // all timelines. See https://github.com/neondatabase/neon/issues/5550 - AttachMarkerMode::Ignore, + SpawnMode::Normal, ctx, - ) { - Ok(tenant) => tenant, - Err(e) => { - error!("Failed to spawn_attach tenant {tenant_id}, reason: {e:#}"); - Tenant::create_broken_tenant(conf, tenant_id, format!("{e:#}")) - } - }; + )?; TenantSlot::Attached(tenant) } @@ -1043,7 +1029,7 @@ pub(crate) async fn load_tenant( location_conf.attach_in_generation(generation); Tenant::persist_tenant_config(conf, &tenant_id, &location_conf).await?; - let new_tenant = schedule_local_tenant_processing(conf, tenant_id, &tenant_path, AttachedTenantConf::try_from(location_conf)?, resources, None, &TENANTS, ctx) + let new_tenant = tenant_spawn(conf, tenant_id, &tenant_path, resources, AttachedTenantConf::try_from(location_conf)?, None, &TENANTS, SpawnMode::Normal, ctx) .with_context(|| { format!("Failed to schedule tenant processing in path {tenant_path:?}") })?; @@ -1117,18 +1103,12 @@ pub(crate) async fn attach_tenant( ) -> Result<(), TenantMapInsertError> { tenant_map_insert(tenant_id, || async { let location_conf = LocationConf::attached_single(tenant_conf, generation); - let tenant_dir = create_tenant_files(conf, &location_conf, &tenant_id, CreateTenantFilesMode::Attach).await?; + let tenant_dir = create_tenant_files(conf, &location_conf, &tenant_id).await?; // TODO: tenant directory remains on disk if we bail out from here on. // See https://github.com/neondatabase/neon/issues/4233 - // Without the attach marker, schedule_local_tenant_processing will treat the attached tenant as fully attached - let marker_file_exists = conf - .tenant_attaching_mark_file_path(&tenant_id) - .try_exists() - .context("check for attach marker file existence")?; - anyhow::ensure!(marker_file_exists, "create_tenant_files should have created the attach marker file"); - - let attached_tenant = schedule_local_tenant_processing(conf, tenant_id, &tenant_dir, AttachedTenantConf::try_from(location_conf)?, resources, None, &TENANTS, ctx)?; + let attached_tenant = tenant_spawn(conf, tenant_id, &tenant_dir, + resources, AttachedTenantConf::try_from(location_conf)?, None, &TENANTS, SpawnMode::Normal, ctx)?; // TODO: tenant object & its background loops remain, untracked in tenant map, if we fail here. // See https://github.com/neondatabase/neon/issues/4233 diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 245649c3ad..d91970a1c0 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -168,36 +168,14 @@ //! - create `Timeline` struct and a `RemoteTimelineClient` //! - initialize the client's upload queue with its `IndexPart` //! - schedule uploads for layers that are only present locally. -//! - if the remote `IndexPart`'s metadata was newer than the metadata in -//! the local filesystem, write the remote metadata to the local filesystem //! - After the above is done for each timeline, open the tenant for business by //! transitioning it from `TenantState::Attaching` to `TenantState::Active` state. //! This starts the timelines' WAL-receivers and the tenant's GC & Compaction loops. //! -//! We keep track of the fact that a client is in `Attaching` state in a marker -//! file on the local disk. This is critical because, when we restart the pageserver, -//! we do not want to do the `List timelines` step for each tenant that has already -//! been successfully attached (for performance & cost reasons). -//! Instead, for a tenant without the attach marker file, we assume that the -//! local state is in sync or ahead of the remote state. This includes the list -//! of all of the tenant's timelines, which is particularly critical to be up-to-date: -//! if there's a timeline on the remote that the pageserver doesn't know about, -//! the GC will not consider its branch point, leading to data loss. -//! So, for a tenant with the attach marker file, we know that we do not yet have -//! persisted all the remote timeline's metadata files locally. To exclude the -//! risk above, we re-run the procedure for such tenants -//! //! # Operating Without Remote Storage //! //! If no remote storage configuration is provided, the [`RemoteTimelineClient`] is //! not created and the uploads are skipped. -//! Theoretically, it should be ok to remove and re-add remote storage configuration to -//! the pageserver config at any time, since it doesn't make a difference to -//! [`Timeline::load_layer_map`]. -//! Of course, the remote timeline dir must not change while we have de-configured -//! remote storage, i.e., the pageserver must remain the owner of the given prefix -//! in remote storage. -//! But note that we don't test any of this right now. //! //! [`Tenant::timeline_init_and_sync`]: super::Tenant::timeline_init_and_sync //! [`Timeline::load_layer_map`]: super::Timeline::load_layer_map @@ -468,7 +446,10 @@ impl RemoteTimelineClient { // /// Download index file - pub async fn download_index_file(&self) -> Result { + pub async fn download_index_file( + &self, + cancel: CancellationToken, + ) -> Result { let _unfinished_gauge_guard = self.metrics.call_begin( &RemoteOpFileKind::Index, &RemoteOpKind::Download, @@ -482,6 +463,7 @@ impl RemoteTimelineClient { &self.tenant_id, &self.timeline_id, self.generation, + cancel, ) .measure_remote_op( self.tenant_id, @@ -1725,7 +1707,11 @@ mod tests { let client = timeline.remote_client.as_ref().unwrap(); // Download back the index.json, and check that the list of files is correct - let initial_index_part = match client.download_index_file().await.unwrap() { + let initial_index_part = match client + .download_index_file(CancellationToken::new()) + .await + .unwrap() + { MaybeDeletedIndexPart::IndexPart(index_part) => index_part, MaybeDeletedIndexPart::Deleted(_) => panic!("unexpectedly got deleted index part"), }; @@ -1814,7 +1800,11 @@ mod tests { } // Download back the index.json, and check that the list of files is correct - let index_part = match client.download_index_file().await.unwrap() { + let index_part = match client + .download_index_file(CancellationToken::new()) + .await + .unwrap() + { MaybeDeletedIndexPart::IndexPart(index_part) => index_part, MaybeDeletedIndexPart::Deleted(_) => panic!("unexpectedly got deleted index part"), }; @@ -2013,7 +2003,7 @@ mod tests { let client = test_state.build_client(get_generation); let download_r = client - .download_index_file() + .download_index_file(CancellationToken::new()) .await .expect("download should always succeed"); assert!(matches!(download_r, MaybeDeletedIndexPart::IndexPart(_))); diff --git a/pageserver/src/tenant/remote_timeline_client/download.rs b/pageserver/src/tenant/remote_timeline_client/download.rs index ca6e3293d6..6039b01ab8 100644 --- a/pageserver/src/tenant/remote_timeline_client/download.rs +++ b/pageserver/src/tenant/remote_timeline_client/download.rs @@ -18,8 +18,8 @@ use crate::config::PageServerConf; use crate::tenant::remote_timeline_client::{remote_layer_path, remote_timelines_path}; use crate::tenant::storage_layer::LayerFileName; use crate::tenant::timeline::span::debug_assert_current_span_has_tenant_and_timeline_id; -use crate::tenant::{Generation, TENANT_DELETED_MARKER_FILE_NAME}; -use remote_storage::{DownloadError, GenericRemoteStorage}; +use crate::tenant::Generation; +use remote_storage::{DownloadError, GenericRemoteStorage, ListingMode}; use utils::crashsafe::path_with_suffix_extension; use utils::id::{TenantId, TimelineId}; @@ -170,53 +170,43 @@ pub fn is_temp_download_file(path: &Utf8Path) -> bool { pub async fn list_remote_timelines( storage: &GenericRemoteStorage, tenant_id: TenantId, -) -> anyhow::Result> { + cancel: CancellationToken, +) -> anyhow::Result<(HashSet, HashSet)> { let remote_path = remote_timelines_path(&tenant_id); fail::fail_point!("storage-sync-list-remote-timelines", |_| { anyhow::bail!("storage-sync-list-remote-timelines"); }); - let timelines = download_retry( - || storage.list_prefixes(Some(&remote_path)), - &format!("list prefixes for {tenant_id}"), + let listing = download_retry_forever( + || storage.list(Some(&remote_path), ListingMode::WithDelimiter), + &format!("list timelines for {tenant_id}"), + cancel, ) .await?; - if timelines.is_empty() { - anyhow::bail!("no timelines found on the remote storage") - } - let mut timeline_ids = HashSet::new(); + let mut other_prefixes = HashSet::new(); - for timeline_remote_storage_key in timelines { - if timeline_remote_storage_key.object_name() == Some(TENANT_DELETED_MARKER_FILE_NAME) { - // A `deleted` key within `timelines/` is a marker file, not a timeline. Ignore it. - // This code will be removed in https://github.com/neondatabase/neon/pull/5580 - continue; - } - + for timeline_remote_storage_key in listing.prefixes { let object_name = timeline_remote_storage_key.object_name().ok_or_else(|| { anyhow::anyhow!("failed to get timeline id for remote tenant {tenant_id}") })?; - let timeline_id: TimelineId = object_name - .parse() - .with_context(|| format!("parse object name into timeline id '{object_name}'"))?; - - // list_prefixes is assumed to return unique names. Ensure this here. - // NB: it's safer to bail out than warn-log this because the pageserver - // needs to absolutely know about _all_ timelines that exist, so that - // GC knows all the branchpoints. If we skipped over a timeline instead, - // GC could delete a layer that's still needed by that timeline. - anyhow::ensure!( - !timeline_ids.contains(&timeline_id), - "list_prefixes contains duplicate timeline id {timeline_id}" - ); - timeline_ids.insert(timeline_id); + match object_name.parse::() { + Ok(t) => timeline_ids.insert(t), + Err(_) => other_prefixes.insert(object_name.to_string()), + }; } - Ok(timeline_ids) + for key in listing.keys { + let object_name = key + .object_name() + .ok_or_else(|| anyhow::anyhow!("object name for key {key}"))?; + other_prefixes.insert(object_name.to_string()); + } + + Ok((timeline_ids, other_prefixes)) } async fn do_download_index_part( @@ -224,10 +214,11 @@ async fn do_download_index_part( tenant_id: &TenantId, timeline_id: &TimelineId, index_generation: Generation, + cancel: CancellationToken, ) -> Result { let remote_path = remote_index_path(tenant_id, timeline_id, index_generation); - let index_part_bytes = download_retry( + let index_part_bytes = download_retry_forever( || async { let mut index_part_download = storage.download(&remote_path).await?; @@ -242,6 +233,7 @@ async fn do_download_index_part( Ok(index_part_bytes) }, &format!("download {remote_path:?}"), + cancel, ) .await?; @@ -263,19 +255,28 @@ pub(super) async fn download_index_part( tenant_id: &TenantId, timeline_id: &TimelineId, my_generation: Generation, + cancel: CancellationToken, ) -> Result { debug_assert_current_span_has_tenant_and_timeline_id(); if my_generation.is_none() { // Operating without generations: just fetch the generation-less path - return do_download_index_part(storage, tenant_id, timeline_id, my_generation).await; + return do_download_index_part(storage, tenant_id, timeline_id, my_generation, cancel) + .await; } // Stale case: If we were intentionally attached in a stale generation, there may already be a remote // index in our generation. // // This is an optimization to avoid doing the listing for the general case below. - let res = do_download_index_part(storage, tenant_id, timeline_id, my_generation).await; + let res = do_download_index_part( + storage, + tenant_id, + timeline_id, + my_generation, + cancel.clone(), + ) + .await; match res { Ok(index_part) => { tracing::debug!( @@ -295,8 +296,14 @@ pub(super) async fn download_index_part( // we want to find the most recent index from a previous generation. // // This is an optimization to avoid doing the listing for the general case below. - let res = - do_download_index_part(storage, tenant_id, timeline_id, my_generation.previous()).await; + let res = do_download_index_part( + storage, + tenant_id, + timeline_id, + my_generation.previous(), + cancel.clone(), + ) + .await; match res { Ok(index_part) => { tracing::debug!("Found index_part from previous generation"); @@ -340,13 +347,14 @@ pub(super) async fn download_index_part( match max_previous_generation { Some(g) => { tracing::debug!("Found index_part in generation {g:?}"); - do_download_index_part(storage, tenant_id, timeline_id, g).await + do_download_index_part(storage, tenant_id, timeline_id, g, cancel).await } None => { // Migration from legacy pre-generation state: we have a generation but no prior // attached pageservers did. Try to load from a no-generation path. tracing::info!("No index_part.json* found"); - do_download_index_part(storage, tenant_id, timeline_id, Generation::none()).await + do_download_index_part(storage, tenant_id, timeline_id, Generation::none(), cancel) + .await } } } @@ -376,3 +384,23 @@ where ) .await } + +async fn download_retry_forever( + op: O, + description: &str, + cancel: CancellationToken, +) -> Result +where + O: FnMut() -> F, + F: Future>, +{ + backoff::retry( + op, + |e| matches!(e, DownloadError::BadInput(_) | DownloadError::NotFound), + FAILED_DOWNLOAD_WARN_THRESHOLD, + u32::MAX, + description, + backoff::Cancel::new(cancel, || DownloadError::Cancelled), + ) + .await +} diff --git a/pageserver/src/tenant/timeline/delete.rs b/pageserver/src/tenant/timeline/delete.rs index 7d55388f44..8cead5ebf2 100644 --- a/pageserver/src/tenant/timeline/delete.rs +++ b/pageserver/src/tenant/timeline/delete.rs @@ -294,6 +294,7 @@ async fn cleanup_remaining_timeline_fs_traces( // Remove delete mark tokio::fs::remove_file(conf.timeline_delete_mark_file_path(tenant_id, timeline_id)) .await + .or_else(fs_ext::ignore_not_found) .context("remove delete mark") } diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 9184f0c43a..d1d19fa542 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -60,6 +60,7 @@ from fixtures.utils import ( allure_attach_from_dir, get_self_dir, subprocess_capture, + wait_until, ) """ @@ -1679,6 +1680,40 @@ class NeonPageserver(PgProtocol): self.running = False return self + def restart(self, immediate: bool = False): + """ + High level wrapper for restart: restarts the process, and waits for + tenant state to stabilize. + """ + self.stop(immediate=immediate) + self.start() + self.quiesce_tenants() + + def quiesce_tenants(self): + """ + Wait for all tenants to enter a stable state (Active or Broken) + + Call this after restarting the pageserver, or after attaching a tenant, + to ensure that it is ready for use. + """ + + stable_states = {"Active", "Broken"} + + client = self.http_client() + + def complete(): + log.info("Checking tenants...") + tenants = client.tenant_list() + log.info(f"Tenant list: {tenants}...") + any_unstable = any((t["state"]["slug"] not in stable_states) for t in tenants) + if any_unstable: + for t in tenants: + log.info(f"Waiting for tenant {t['id']} in state {t['state']['slug']}") + log.info(f"any_unstable={any_unstable}") + assert not any_unstable + + wait_until(20, 0.5, complete) + def __enter__(self) -> "NeonPageserver": return self diff --git a/test_runner/regress/test_branching.py b/test_runner/regress/test_branching.py index 32b1466c90..c4f743204e 100644 --- a/test_runner/regress/test_branching.py +++ b/test_runner/regress/test_branching.py @@ -333,16 +333,30 @@ def test_non_uploaded_root_timeline_is_deleted_after_restart(neon_env_builder: N env = neon_env_builder.init_configs() env.start() - env.pageserver.allowed_errors.append( - ".*request{method=POST path=/v1/tenant/.*/timeline request_id=.*}: request was dropped before completing.*" + env.pageserver.allowed_errors.extend( + [ + ".*request{method=POST path=/v1/tenant/.*/timeline request_id=.*}: request was dropped before completing.*", + ".*Failed to load index_part from remote storage.*", + # On a fast restart, there may be an initdb still running in a basebackup...__temp directory + ".*Failed to purge.*Directory not empty.*", + ] ) ps_http = env.pageserver.http_client() # pause all uploads - ps_http.configure_failpoints(("before-upload-index-pausable", "pause")) ps_http.tenant_create(env.initial_tenant) + # Create a timeline whose creation will succeed. The tenant will need at least one + # timeline to be loadable. + success_timeline = TimelineId.generate() + log.info(f"Creating timeline {success_timeline}") + ps_http.timeline_create(env.pg_version, env.initial_tenant, success_timeline, timeout=60) + + # Create a timeline whose upload to remote storage will be blocked + ps_http.configure_failpoints(("before-upload-index-pausable", "pause")) + def start_creating_timeline(): + log.info(f"Creating (expect failure) timeline {env.initial_timeline}") with pytest.raises(RequestException): ps_http.timeline_create( env.pg_version, env.initial_tenant, env.initial_timeline, timeout=60 @@ -366,6 +380,9 @@ def test_non_uploaded_root_timeline_is_deleted_after_restart(neon_env_builder: N with pytest.raises(PageserverApiException, match="not found"): ps_http.timeline_detail(env.initial_tenant, env.initial_timeline) + # The one successfully created timeline should still be there. + assert len(ps_http.timeline_list(tenant_id=env.initial_tenant)) == 1 + def test_non_uploaded_branch_is_deleted_after_restart(neon_env_builder: NeonEnvBuilder): """ diff --git a/test_runner/regress/test_broken_timeline.py b/test_runner/regress/test_broken_timeline.py index 48accdb43d..b1b47b3f2c 100644 --- a/test_runner/regress/test_broken_timeline.py +++ b/test_runner/regress/test_broken_timeline.py @@ -15,7 +15,7 @@ from fixtures.types import TenantId, TimelineId # Test restarting page server, while safekeeper and compute node keep # running. -def test_broken_timeline(neon_env_builder: NeonEnvBuilder): +def test_local_corruption(neon_env_builder: NeonEnvBuilder): env = neon_env_builder.init_start() env.pageserver.allowed_errors.extend( @@ -69,24 +69,19 @@ def test_broken_timeline(neon_env_builder: NeonEnvBuilder): env.pageserver.start() - # Tenant 0 should still work + # Un-damaged tenant works pg0.start() assert pg0.safe_psql("SELECT COUNT(*) FROM t")[0][0] == 100 - # But all others are broken - - # First timeline would not get loaded into pageserver due to corrupt metadata file - with pytest.raises( - Exception, match=f"Tenant {tenant1} will not become active. Current state: Broken" - ) as err: - pg1.start() - log.info( - f"As expected, compute startup failed eagerly for timeline with corrupt metadata: {err}" - ) + # Tenant with corrupt local metadata works: remote storage is authoritative for metadata + pg1.start() + assert pg1.safe_psql("SELECT COUNT(*) FROM t")[0][0] == 100 # Second timeline will fail during basebackup, because the local layer file is corrupt. # It will fail when we try to read (and reconstruct) a page from it, ergo the error message. # (We don't check layer file contents on startup, when loading the timeline) + # + # This will change when we implement checksums for layers with pytest.raises(Exception, match="layer loading failed:") as err: pg2.start() log.info( @@ -133,8 +128,7 @@ def test_timeline_init_break_before_checkpoint(neon_env_builder: NeonEnvBuilder) _ = env.neon_cli.create_timeline("test_timeline_init_break_before_checkpoint", tenant_id) # Restart the page server - env.pageserver.stop(immediate=True) - env.pageserver.start() + env.pageserver.restart(immediate=True) # Creating the timeline didn't finish. The other timelines on tenant should still be present and work normally. new_tenant_timelines = env.neon_cli.list_timelines(tenant_id) diff --git a/test_runner/regress/test_pageserver_restart.py b/test_runner/regress/test_pageserver_restart.py index b49a1a40dd..4c51155236 100644 --- a/test_runner/regress/test_pageserver_restart.py +++ b/test_runner/regress/test_pageserver_restart.py @@ -62,14 +62,14 @@ def test_pageserver_restart(neon_env_builder: NeonEnvBuilder, generations: bool) tenant_load_delay_ms = 5000 env.pageserver.stop() env.pageserver.start( - extra_env_vars={"FAILPOINTS": f"before-loading-tenant=return({tenant_load_delay_ms})"} + extra_env_vars={"FAILPOINTS": f"before-attaching-tenant=return({tenant_load_delay_ms})"} ) - # Check that it's in Loading state + # Check that it's in Attaching state client = env.pageserver.http_client() tenant_status = client.tenant_status(env.initial_tenant) log.info("Tenant status : %s", tenant_status) - assert tenant_status["state"]["slug"] == "Loading" + assert tenant_status["state"]["slug"] == "Attaching" # Try to read. This waits until the loading finishes, and then return normally. cur.execute("SELECT count(*) FROM foo") diff --git a/test_runner/regress/test_tenant_delete.py b/test_runner/regress/test_tenant_delete.py index 7a0b2694b8..ae77197088 100644 --- a/test_runner/regress/test_tenant_delete.py +++ b/test_runner/regress/test_tenant_delete.py @@ -241,8 +241,7 @@ def test_delete_tenant_exercise_crash_safety_failpoints( assert reason.endswith(f"failpoint: {failpoint}"), reason if check is Check.RETRY_WITH_RESTART: - env.pageserver.stop() - env.pageserver.start() + env.pageserver.restart() if failpoint in ( "tenant-delete-before-shutdown", diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py index e92a906fab..11e8a80e1d 100644 --- a/test_runner/regress/test_tenant_detach.py +++ b/test_runner/regress/test_tenant_detach.py @@ -66,10 +66,6 @@ def test_tenant_reattach( env.pageserver.allowed_errors.append( f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" ) - # Thats because of UnreliableWrapper's injected failures - env.pageserver.allowed_errors.append( - f".*failed to fetch tenant deletion mark at tenants/({tenant_id}|{env.initial_tenant})/deleted attempt 1.*" - ) with env.endpoints.create_start("main", tenant_id=tenant_id) as endpoint: with endpoint.cursor() as cur: @@ -116,7 +112,7 @@ def test_tenant_reattach( assert query_scalar(cur, "SELECT count(*) FROM t") == 100000 # Check that we had to retry the downloads - assert env.pageserver.log_contains(".*list prefixes.*failed, will retry.*") + assert env.pageserver.log_contains(".*list timelines.*failed, will retry.*") assert env.pageserver.log_contains(".*download.*failed, will retry.*") @@ -643,47 +639,6 @@ def test_ignored_tenant_download_missing_layers(neon_env_builder: NeonEnvBuilder ensure_test_data(data_id, data_secret, endpoint) -# Tests that it's possible to `load` broken tenants: -# * `ignore` a tenant -# * removes its `metadata` file locally -# * `load` the same tenant -# * ensure that it's status is `Broken` -def test_ignored_tenant_stays_broken_without_metadata(neon_env_builder: NeonEnvBuilder): - neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) - env = neon_env_builder.init_start() - pageserver_http = env.pageserver.http_client() - env.endpoints.create_start("main") - - tenant_id = env.initial_tenant - timeline_id = env.initial_timeline - - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") - env.pageserver.allowed_errors.append( - f".*Tenant {tenant_id} will not become active\\. Current state: (Broken|Stopping).*" - ) - - # ignore the tenant and remove its metadata - pageserver_http.tenant_ignore(tenant_id) - timeline_dir = env.pageserver.timeline_dir(tenant_id, timeline_id) - metadata_removed = False - for dir_entry in timeline_dir.iterdir(): - if dir_entry.name == "metadata": - # Looks like a layer file. Remove it - dir_entry.unlink() - metadata_removed = True - assert metadata_removed, f"Failed to find metadata file in {timeline_dir}" - - env.pageserver.allowed_errors.append( - f".*{tenant_id}.*: load failed.*: failed to load metadata.*" - ) - - # now, load it from the local files and expect it to be broken due to inability to load tenant files into memory - pageserver_http.tenant_load(tenant_id=tenant_id) - wait_until_tenant_state(pageserver_http, tenant_id, "Broken", 5) - - # Tests that attach is never working on a tenant, ignored or not, as long as it's not absent locally # Similarly, tests that it's not possible to schedule a `load` for tenat that's not ignored. def test_load_attach_negatives(neon_env_builder: NeonEnvBuilder): @@ -778,7 +733,8 @@ def test_ignore_while_attaching( tenants_before_ignore ), "Only ignored tenant should be missing" - # But can load it from local files, that will restore attach. + # Calling load will bring the tenant back online + pageserver_http.configure_failpoints([("attach-before-activate", "off")]) pageserver_http.tenant_load(tenant_id) wait_until_tenant_state(pageserver_http, tenant_id, "Active", 5) diff --git a/test_runner/regress/test_tenants.py b/test_runner/regress/test_tenants.py index 40dff194aa..090d586721 100644 --- a/test_runner/regress/test_tenants.py +++ b/test_runner/regress/test_tenants.py @@ -1,5 +1,4 @@ import os -import shutil import time from contextlib import closing from datetime import datetime @@ -20,7 +19,7 @@ from fixtures.neon_fixtures import ( ) from fixtures.pageserver.utils import timeline_delete_wait_completed from fixtures.remote_storage import RemoteStorageKind, available_remote_storages -from fixtures.types import Lsn, TenantId, TimelineId +from fixtures.types import Lsn, TenantId from fixtures.utils import wait_until from prometheus_client.samples import Sample @@ -298,13 +297,8 @@ def test_pageserver_with_empty_tenants( client = env.pageserver.http_client() - tenant_with_empty_timelines = TenantId.generate() - client.tenant_create(tenant_with_empty_timelines) - temp_timelines = client.timeline_list(tenant_with_empty_timelines) - for temp_timeline in temp_timelines: - timeline_delete_wait_completed( - client, tenant_with_empty_timelines, TimelineId(temp_timeline["timeline_id"]) - ) + tenant_with_empty_timelines = env.initial_tenant + timeline_delete_wait_completed(client, tenant_with_empty_timelines, env.initial_timeline) files_in_timelines_dir = sum( 1 for _p in Path.iterdir(env.pageserver.timeline_dir(tenant_with_empty_timelines)) @@ -317,34 +311,19 @@ def test_pageserver_with_empty_tenants( env.endpoints.stop_all() env.pageserver.stop() - tenant_without_timelines_dir = env.initial_tenant - shutil.rmtree(env.pageserver.timeline_dir(tenant_without_timelines_dir)) - env.pageserver.start() client = env.pageserver.http_client() - def not_loading(): + def not_attaching(): tenants = client.tenant_list() - assert len(tenants) == 2 - assert all(t["state"]["slug"] != "Loading" for t in tenants) + assert len(tenants) == 1 + assert all(t["state"]["slug"] != "Attaching" for t in tenants) - wait_until(10, 0.2, not_loading) + wait_until(10, 0.2, not_attaching) tenants = client.tenant_list() - [broken_tenant] = [t for t in tenants if t["id"] == str(tenant_without_timelines_dir)] - assert ( - broken_tenant["state"]["slug"] == "Broken" - ), f"Tenant {tenant_without_timelines_dir} without timelines dir should be broken" - - broken_tenant_status = client.tenant_status(tenant_without_timelines_dir) - assert ( - broken_tenant_status["state"]["slug"] == "Broken" - ), f"Tenant {tenant_without_timelines_dir} without timelines dir should be broken" - - assert env.pageserver.log_contains(".*load failed, setting tenant state to Broken:.*") - [loaded_tenant] = [t for t in tenants if t["id"] == str(tenant_with_empty_timelines)] assert ( loaded_tenant["state"]["slug"] == "Active" @@ -358,9 +337,6 @@ def test_pageserver_with_empty_tenants( time.sleep(1) # to allow metrics propagation ps_metrics = client.get_metrics() - broken_tenants_metric_filter = { - "tenant_id": str(tenant_without_timelines_dir), - } active_tenants_metric_filter = { "state": "Active", } @@ -374,13 +350,3 @@ def test_pageserver_with_empty_tenants( assert ( tenant_active_count == 1 ), f"Tenant {tenant_with_empty_timelines} should have metric as active" - - tenant_broken_count = int( - ps_metrics.query_one( - "pageserver_broken_tenants_count", filter=broken_tenants_metric_filter - ).value - ) - - assert ( - tenant_broken_count == 1 - ), f"Tenant {tenant_without_timelines_dir} should have metric as broken" diff --git a/test_runner/regress/test_threshold_based_eviction.py b/test_runner/regress/test_threshold_based_eviction.py index 12866accc7..27d5cce5f2 100644 --- a/test_runner/regress/test_threshold_based_eviction.py +++ b/test_runner/regress/test_threshold_based_eviction.py @@ -70,8 +70,7 @@ def test_threshold_based_eviction( } # restart because changing tenant config is not instant - env.pageserver.stop() - env.pageserver.start() + env.pageserver.restart() assert ps_http.tenant_config(tenant_id).effective_config["eviction_policy"] == { "kind": "LayerAccessThreshold", diff --git a/test_runner/regress/test_timeline_delete.py b/test_runner/regress/test_timeline_delete.py index c412809a3a..2e1fcd38fe 100644 --- a/test_runner/regress/test_timeline_delete.py +++ b/test_runner/regress/test_timeline_delete.py @@ -277,13 +277,6 @@ def test_delete_timeline_exercise_crash_safety_failpoints( if failpoint == "timeline-delete-after-index-delete": m = ps_http.get_metrics() - assert ( - m.query_one( - "remote_storage_s3_request_seconds_count", - filter={"request_type": "get_object", "result": "err"}, - ).value - == 2 # One is missing tenant deletion mark, second is missing index part - ) assert ( m.query_one( "remote_storage_s3_request_seconds_count", From ba4fe9e10f62dea18e5aa0f26b43a60195397818 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 26 Oct 2023 14:54:52 +0100 Subject: [PATCH 28/69] pageserver: fix the second "AUX files" warning (#5673) In https://github.com/neondatabase/neon/pull/5669 I didn't notice that the same warning is logged in two places: fix the other one. --- pageserver/src/pgdatadir_mapping.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index b611583b60..daadf6abd4 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -552,7 +552,8 @@ impl Timeline { Err(e) => Err(PageReconstructError::from(e)), }, Err(e) => { - warn!("Failed to get info about AUX files: {}", e); + // This is expected: historical databases do not have the key. + debug!("Failed to get info about AUX files: {}", e); Ok(HashMap::new()) } } From 116c342cada0fc7f29b49105ee48bc72e8f246c6 Mon Sep 17 00:00:00 2001 From: Sasha Krassovsky Date: Thu, 26 Oct 2023 09:56:07 -0800 Subject: [PATCH 29/69] Support changing pageserver dynamically (#5542) ## Problem We currently require full restart of compute if we change the pageserver url ## Summary of changes Makes it so that we don't have to do a full restart, but can just send SIGHUP --- compute_tools/src/bin/compute_ctl.rs | 1 + control_plane/src/bin/neon_local.rs | 24 ++++ control_plane/src/endpoint.rs | 111 ++++++++++++++---- pgxn/neon/libpagestore.c | 21 +++- test_runner/fixtures/neon_fixtures.py | 18 +++ test_runner/regress/test_change_pageserver.py | 64 ++++++++++ 6 files changed, 216 insertions(+), 23 deletions(-) create mode 100644 test_runner/regress/test_change_pageserver.py diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index 2fe07c4287..04ac077c7b 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -156,6 +156,7 @@ fn main() -> Result<()> { let path = Path::new(sp); let file = File::open(path)?; spec = Some(serde_json::from_reader(file)?); + live_config_allowed = true; } else if let Some(id) = compute_id { if let Some(cp_base) = control_plane_uri { live_config_allowed = true; diff --git a/control_plane/src/bin/neon_local.rs b/control_plane/src/bin/neon_local.rs index d43566d2df..bc68eeaa8a 100644 --- a/control_plane/src/bin/neon_local.rs +++ b/control_plane/src/bin/neon_local.rs @@ -798,6 +798,24 @@ fn handle_endpoint(ep_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<( ep.start(&auth_token, safekeepers, remote_ext_config)?; } } + "reconfigure" => { + let endpoint_id = sub_args + .get_one::("endpoint_id") + .ok_or_else(|| anyhow!("No endpoint ID provided to reconfigure"))?; + let endpoint = cplane + .endpoints + .get(endpoint_id.as_str()) + .with_context(|| format!("postgres endpoint {endpoint_id} is not found"))?; + let pageserver_id = + if let Some(id_str) = sub_args.get_one::("endpoint-pageserver-id") { + Some(NodeId( + id_str.parse().context("while parsing pageserver id")?, + )) + } else { + None + }; + endpoint.reconfigure(pageserver_id)?; + } "stop" => { let endpoint_id = sub_args .get_one::("endpoint_id") @@ -1369,6 +1387,12 @@ fn cli() -> Command { .arg(safekeepers_arg) .arg(remote_ext_config_args) ) + .subcommand(Command::new("reconfigure") + .about("Reconfigure the endpoint") + .arg(endpoint_pageserver_id_arg) + .arg(endpoint_id_arg.clone()) + .arg(tenant_id_arg.clone()) + ) .subcommand( Command::new("stop") .arg(endpoint_id_arg) diff --git a/control_plane/src/endpoint.rs b/control_plane/src/endpoint.rs index acd9061664..cb16f48829 100644 --- a/control_plane/src/endpoint.rs +++ b/control_plane/src/endpoint.rs @@ -414,18 +414,34 @@ impl Endpoint { ); } - // Also wait for the compute_ctl process to die. It might have some cleanup - // work to do after postgres stops, like syncing safekeepers, etc. - // + Ok(()) + } + + fn wait_for_compute_ctl_to_exit(&self) -> Result<()> { // TODO use background_process::stop_process instead let pidfile_path = self.endpoint_path().join("compute_ctl.pid"); let pid: u32 = std::fs::read_to_string(pidfile_path)?.parse()?; let pid = nix::unistd::Pid::from_raw(pid as i32); crate::background_process::wait_until_stopped("compute_ctl", pid)?; - Ok(()) } + fn read_postgresql_conf(&self) -> Result { + // Slurp the endpoints//postgresql.conf file into + // memory. We will include it in the spec file that we pass to + // `compute_ctl`, and `compute_ctl` will write it to the postgresql.conf + // in the data directory. + let postgresql_conf_path = self.endpoint_path().join("postgresql.conf"); + match std::fs::read(&postgresql_conf_path) { + Ok(content) => Ok(String::from_utf8(content)?), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok("".to_string()), + Err(e) => Err(anyhow::Error::new(e).context(format!( + "failed to read config file in {}", + postgresql_conf_path.to_str().unwrap() + ))), + } + } + pub fn start( &self, auth_token: &Option, @@ -436,21 +452,7 @@ impl Endpoint { anyhow::bail!("The endpoint is already running"); } - // Slurp the endpoints//postgresql.conf file into - // memory. We will include it in the spec file that we pass to - // `compute_ctl`, and `compute_ctl` will write it to the postgresql.conf - // in the data directory. - let postgresql_conf_path = self.endpoint_path().join("postgresql.conf"); - let postgresql_conf = match std::fs::read(&postgresql_conf_path) { - Ok(content) => String::from_utf8(content)?, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => "".to_string(), - Err(e) => { - return Err(anyhow::Error::new(e).context(format!( - "failed to read config file in {}", - postgresql_conf_path.to_str().unwrap() - ))) - } - }; + let postgresql_conf = self.read_postgresql_conf()?; // We always start the compute node from scratch, so if the Postgres // data dir exists from a previous launch, remove it first. @@ -621,6 +623,61 @@ impl Endpoint { } } + pub fn reconfigure(&self, pageserver_id: Option) -> Result<()> { + let mut spec: ComputeSpec = { + let spec_path = self.endpoint_path().join("spec.json"); + let file = std::fs::File::open(spec_path)?; + serde_json::from_reader(file)? + }; + + let postgresql_conf = self.read_postgresql_conf()?; + spec.cluster.postgresql_conf = Some(postgresql_conf); + + if let Some(pageserver_id) = pageserver_id { + let endpoint_config_path = self.endpoint_path().join("endpoint.json"); + let mut endpoint_conf: EndpointConf = { + let file = std::fs::File::open(&endpoint_config_path)?; + serde_json::from_reader(file)? + }; + endpoint_conf.pageserver_id = pageserver_id; + std::fs::write( + endpoint_config_path, + serde_json::to_string_pretty(&endpoint_conf)?, + )?; + + let pageserver = + PageServerNode::from_env(&self.env, self.env.get_pageserver_conf(pageserver_id)?); + let ps_http_conf = &pageserver.pg_connection_config; + let (host, port) = (ps_http_conf.host(), ps_http_conf.port()); + spec.pageserver_connstring = Some(format!("postgresql://no_user@{host}:{port}")); + } + + let client = reqwest::blocking::Client::new(); + let response = client + .post(format!( + "http://{}:{}/configure", + self.http_address.ip(), + self.http_address.port() + )) + .body(format!( + "{{\"spec\":{}}}", + serde_json::to_string_pretty(&spec)? + )) + .send()?; + + let status = response.status(); + if !(status.is_client_error() || status.is_server_error()) { + Ok(()) + } else { + let url = response.url().to_owned(); + let msg = match response.text() { + Ok(err_body) => format!("Error: {}", err_body), + Err(_) => format!("Http error ({}) at {}.", status.as_u16(), url), + }; + Err(anyhow::anyhow!(msg)) + } + } + pub fn stop(&self, destroy: bool) -> Result<()> { // If we are going to destroy data directory, // use immediate shutdown mode, otherwise, @@ -629,15 +686,25 @@ impl Endpoint { // Postgres is always started from scratch, so stop // without destroy only used for testing and debugging. // + self.pg_ctl( + if destroy { + &["-m", "immediate", "stop"] + } else { + &["stop"] + }, + &None, + )?; + + // Also wait for the compute_ctl process to die. It might have some cleanup + // work to do after postgres stops, like syncing safekeepers, etc. + // + self.wait_for_compute_ctl_to_exit()?; if destroy { - self.pg_ctl(&["-m", "immediate", "stop"], &None)?; println!( "Destroying postgres data directory '{}'", self.pgdata().to_str().unwrap() ); std::fs::remove_dir_all(self.endpoint_path())?; - } else { - self.pg_ctl(&["stop"], &None)?; } Ok(()) } diff --git a/pgxn/neon/libpagestore.c b/pgxn/neon/libpagestore.c index ca24ec7586..0a944a6bf0 100644 --- a/pgxn/neon/libpagestore.c +++ b/pgxn/neon/libpagestore.c @@ -19,6 +19,7 @@ #include "access/xlog.h" #include "access/xlogutils.h" #include "storage/buf_internals.h" +#include "c.h" #include "libpq-fe.h" #include "libpq/pqformat.h" @@ -63,6 +64,21 @@ int max_reconnect_attempts = 60; bool (*old_redo_read_buffer_filter) (XLogReaderState *record, uint8 block_id) = NULL; static bool pageserver_flush(void); +static void pageserver_disconnect(void); + + +static pqsigfunc prev_signal_handler; + +static void +pageserver_sighup_handler(SIGNAL_ARGS) +{ + if (prev_signal_handler) + { + prev_signal_handler(postgres_signal_arg); + } + neon_log(LOG, "Received SIGHUP, disconnecting pageserver. New pageserver connstring is %s", page_server_connstring); + pageserver_disconnect(); +} static bool pageserver_connect(int elevel) @@ -400,7 +416,7 @@ pg_init_libpagestore(void) NULL, &page_server_connstring, "", - PGC_POSTMASTER, + PGC_SIGHUP, 0, /* no flags required */ NULL, NULL, NULL); @@ -482,5 +498,8 @@ pg_init_libpagestore(void) old_redo_read_buffer_filter = redo_read_buffer_filter; redo_read_buffer_filter = neon_redo_read_buffer_filter; } + + prev_signal_handler = pqsignal(SIGHUP, pageserver_sighup_handler); + lfc_init(); } diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index d1d19fa542..7a3e0a9efc 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1445,6 +1445,20 @@ class NeonCli(AbstractNeonCli): res.check_returncode() return res + def endpoint_reconfigure( + self, + endpoint_id: str, + tenant_id: Optional[TenantId] = None, + pageserver_id: Optional[int] = None, + check_return_code=True, + ) -> "subprocess.CompletedProcess[str]": + args = ["endpoint", "reconfigure", endpoint_id] + if tenant_id is not None: + args.extend(["--tenant-id", str(tenant_id)]) + if pageserver_id is not None: + args.extend(["--pageserver-id", str(pageserver_id)]) + return self.raw_cli(args, check_return_code=check_return_code) + def endpoint_stop( self, endpoint_id: str, @@ -2534,6 +2548,10 @@ class Endpoint(PgProtocol): return self + def reconfigure(self, pageserver_id: Optional[int] = None): + assert self.endpoint_id is not None + self.env.neon_cli.endpoint_reconfigure(self.endpoint_id, self.tenant_id, pageserver_id) + def respec(self, **kwargs): """Update the endpoint.json file used by control_plane.""" # Read config diff --git a/test_runner/regress/test_change_pageserver.py b/test_runner/regress/test_change_pageserver.py new file mode 100644 index 0000000000..31092b70b9 --- /dev/null +++ b/test_runner/regress/test_change_pageserver.py @@ -0,0 +1,64 @@ +from fixtures.log_helper import log +from fixtures.neon_fixtures import NeonEnvBuilder +from fixtures.remote_storage import RemoteStorageKind + + +def test_change_pageserver(neon_env_builder: NeonEnvBuilder): + neon_env_builder.num_pageservers = 2 + neon_env_builder.enable_pageserver_remote_storage( + remote_storage_kind=RemoteStorageKind.MOCK_S3, + ) + env = neon_env_builder.init_start() + + env.neon_cli.create_branch("test_change_pageserver") + endpoint = env.endpoints.create_start("test_change_pageserver") + + alt_pageserver_id = env.pageservers[1].id + env.pageservers[1].tenant_attach(env.initial_tenant) + + pg_conn = endpoint.connect() + cur = pg_conn.cursor() + + # Create table, and insert some rows. Make it big enough that it doesn't fit in + # shared_buffers, otherwise the SELECT after restart will just return answer + # from shared_buffers without hitting the page server, which defeats the point + # of this test. + cur.execute("CREATE TABLE foo (t text)") + cur.execute( + """ + INSERT INTO foo + SELECT 'long string to consume some space' || g + FROM generate_series(1, 100000) g + """ + ) + + # Verify that the table is larger than shared_buffers + cur.execute( + """ + select setting::int * pg_size_bytes(unit) as shared_buffers, pg_relation_size('foo') as tbl_size + from pg_settings where name = 'shared_buffers' + """ + ) + row = cur.fetchone() + assert row is not None + log.info(f"shared_buffers is {row[0]}, table size {row[1]}") + assert int(row[0]) < int(row[1]) + + cur.execute("SELECT count(*) FROM foo") + assert cur.fetchone() == (100000,) + + endpoint.reconfigure(pageserver_id=alt_pageserver_id) + + # Verify that the neon.pageserver_connstring GUC is set to the correct thing + cur.execute("SELECT setting FROM pg_settings WHERE name='neon.pageserver_connstring'") + connstring = cur.fetchone() + assert connstring is not None + expected_connstring = f"postgresql://no_user:@localhost:{env.pageservers[1].service_port.pg}" + assert expected_connstring == expected_connstring + + env.pageservers[ + 0 + ].stop() # Stop the old pageserver just to make sure we're reading from the new one + + cur.execute("SELECT count(*) FROM foo") + assert cur.fetchone() == (100000,) From 39b148b74e4d695a8aebd0e7ac485806ec54b406 Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Thu, 26 Oct 2023 12:04:57 -0700 Subject: [PATCH 30/69] Bump vm-builder v0.18.2 -> v0.18.4 (#5666) Only applicable change was neondatabase/autoscaling#584, setting pgbouncer auth_dbname=postgres in order to fix superuser connections from preventing dropping databases. --- .github/workflows/build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 57b5776b76..e0f8afc954 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -847,7 +847,7 @@ jobs: run: shell: sh -eu {0} env: - VM_BUILDER_VERSION: v0.18.2 + VM_BUILDER_VERSION: v0.18.5 steps: - name: Checkout From 7c16b5215e8513b1ddf1de1d21ed6fcb053bd00f Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 26 Oct 2023 20:36:28 +0100 Subject: [PATCH 31/69] scrubber: add separate find/purge garbage commands (#5409) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem The previous garbage cleanup functionality relied on doing a dry run, inspecting logs, and then doing a deletion. This isn't ideal, because what one actually deletes might not be the same as what one saw in the dry run. It's also risky UX to rely on presence/absence of one CLI flag to control deletion: ideally the deletion command should be totally separate from the one that scans the bucket. Related: https://github.com/neondatabase/neon/issues/5037 ## Summary of changes This is a major re-work of the code, which results in a net decrease in line count of about 600. The old code for removing garbage was build around the idea of doing discovery and purging together: a "delete_batch_producer" sent batches into a deleter. The new code writes out both procedures separately, in functions that use the async streams introduced in https://github.com/neondatabase/neon/pull/5176 to achieve fast concurrent access to S3 while retaining the readability of a single function. - Add `find-garbage`, which writes out a JSON file of tenants/timelines to purge - Add `purge-garbage` which consumes the garbage JSON file, applies some extra validations, and does deletions. - The purge command will refuse to execute if the garbage file indicates that only garbage was found: this guards against classes of bugs where the scrubber might incorrectly deem everything garbage. - The purge command defaults to only deleting tenants that were found in "deleted" state in the control plane. This guards against the risk that using the wrong console API endpoint could cause all tenants to appear to be missing. Outstanding work for a future PR: - Make whatever changes are needed to adapt to the Console/Control Plane separation. - Make purge even safer by checking S3 `Modified` times for index_part.json files (not doing this here, because it will depend on the generation-aware changes for finding index_part.json files) ## Checklist before requesting a review - [ ] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. ## Checklist before merging - [ ] Do not forget to reformat commit message to not include the above checklist --------- Co-authored-by: Arpad Müller Co-authored-by: Shany Pozin --- s3_scrubber/README.md | 71 +-- s3_scrubber/src/checks.rs | 183 +------- s3_scrubber/src/cloud_admin_api.rs | 130 +++++- s3_scrubber/src/delete_batch_producer.rs | 354 -------------- .../src/delete_batch_producer/tenant_batch.rs | 87 ---- .../delete_batch_producer/timeline_batch.rs | 102 ---- s3_scrubber/src/garbage.rs | 419 +++++++++++++++++ s3_scrubber/src/lib.rs | 77 +++- s3_scrubber/src/main.rs | 242 ++-------- s3_scrubber/src/metadata_stream.rs | 35 +- s3_scrubber/src/s3_deletion.rs | 434 ------------------ s3_scrubber/src/scan_metadata.rs | 26 +- 12 files changed, 727 insertions(+), 1433 deletions(-) delete mode 100644 s3_scrubber/src/delete_batch_producer.rs delete mode 100644 s3_scrubber/src/delete_batch_producer/tenant_batch.rs delete mode 100644 s3_scrubber/src/delete_batch_producer/timeline_batch.rs create mode 100644 s3_scrubber/src/garbage.rs delete mode 100644 s3_scrubber/src/s3_deletion.rs diff --git a/s3_scrubber/README.md b/s3_scrubber/README.md index 48be3512b4..2f21b9f191 100644 --- a/s3_scrubber/README.md +++ b/s3_scrubber/README.md @@ -25,57 +25,64 @@ _This section is only relevant if using a command that requires access to Neon's ### Commands -#### `tidy` +#### `find-garbage` -Iterate over S3 buckets for storage nodes, checking their contents and removing the data not present in the console. Node S3 data that's not removed is then further checked for discrepancies and, sometimes, validated. - -Unless the global `--delete` argument is provided, this command only dry-runs and logs -what it would have deleted. - -``` -tidy --node-kind= [--depth=] [--skip-validation] -``` +Walk an S3 bucket and cross-reference the contents with the Console API to identify data for +tenants or timelines that should no longer exist. - `--node-kind`: whether to inspect safekeeper or pageserver bucket prefix - `--depth`: whether to only search for deletable tenants, or also search for deletable timelines within active tenants. Default: `tenant` -- `--skip-validation`: skip additional post-deletion checks. Default: `false` +- `--output-path`: filename to write garbage list to. Default `garbage.json` -For a selected S3 path, the tool lists the S3 bucket given for either tenants or both tenants and timelines — for every found entry, console API is queried: any deleted or missing in the API entity is scheduled for deletion from S3. +This command outputs a JSON file describing tenants and timelines to remove, for subsequent +processing by the `purge-garbage` subcommand. -If validation is enabled, only the non-deleted tenants' ones are checked. -For pageserver, timelines' index_part.json on S3 is also checked for various discrepancies: no files are removed, even if there are "extra" S3 files not present in index_part.json: due to the way pageserver updates the remote storage, it's better to do such removals manually, stopping the corresponding tenant first. +**Note that the garbage list format is not stable. The output of `find-garbage` is only + intended for use by the exact same version of the tool running `purge-garbage`** -Command examples: +Example: -`env SSO_ACCOUNT_ID=369495373322 REGION=eu-west-1 BUCKET=neon-dev-storage-eu-west-1 CLOUD_ADMIN_API_TOKEN=${NEON_CLOUD_ADMIN_API_STAGING_KEY} CLOUD_ADMIN_API_URL=[url] cargo run --release -- tidy --node-kind=safekeeper` +`env SSO_ACCOUNT_ID=123456 REGION=eu-west-1 BUCKET=my-dev-bucket CLOUD_ADMIN_API_TOKEN=${NEON_CLOUD_ADMIN_API_STAGING_KEY} CLOUD_ADMIN_API_URL=[url] cargo run --release -- find-garbage --node-kind=pageserver --depth=tenant --output-path=eu-west-1-garbage.json` -`env SSO_ACCOUNT_ID=369495373322 REGION=us-east-2 BUCKET=neon-staging-storage-us-east-2 CLOUD_ADMIN_API_TOKEN=${NEON_CLOUD_ADMIN_API_STAGING_KEY} CLOUD_ADMIN_API_URL=[url] cargo run --release -- tidy --node-kind=pageserver --depth=timeline` +#### `purge-garbage` -When dry run stats look satisfying, use `-- --delete` before the `tidy` command to -disable dry run and run the binary with deletion enabled. +Consume a garbage list from `find-garbage`, and delete the related objects in the S3 bucket. -See these lines (and lines around) in the logs for the final stats: +- `--input-path`: filename to read garbage list from. Default `garbage.json`. +- `--mode`: controls whether to purge only garbage that was specifically marked + deleted in the control plane (`deletedonly`), or also to purge tenants/timelines + that were not present in the control plane at all (`deletedandmissing`) -- `Finished listing the bucket for tenants` -- `Finished active tenant and timeline validation` -- `Total tenant deletion stats` -- `Total timeline deletion stats` +This command learns region/bucket details from the garbage file, so it is not necessary +to pass them on the command line -## Current implementation details +Example: -- The tool does not have any peristent state currently: instead, it creates very verbose logs, with every S3 delete request logged, every tenant/timeline id check, etc. - Worse, any panic or early errored tasks might force the tool to exit without printing the final summary — all affected ids will still be in the logs though. The tool has retries inside it, so it's error-resistant up to some extent, and recent runs showed no traces of errors/panics. +`env SSO_ACCOUNT_ID=123456 cargo run --release -- purge-garbage --node-kind=pageserver --depth=tenant --input-path=eu-west-1-garbage.json` -- Instead of checking non-deleted tenants' timelines instantly, the tool attempts to create separate tasks (futures) for that, - complicating the logic and slowing down the process, this should be fixed and done in one "task". +Add the `--delete` argument before `purge-garbage` to enable deletion. This is intentionally +not provided inline in the example above to avoid accidents. Without the `--delete` flag +the purge command will log all the keys that it would have deleted. -- The tool does uses only publicly available remote resources (S3, console) and does not access pageserver/safekeeper nodes themselves. - Yet, its S3 set up should be prepared for running on any pageserver/safekeeper node, using node's S3 credentials, so the node API access logic could be implemented relatively simply on top. +#### `scan-metadata` -## Cleanup procedure: +Walk objects in a pageserver S3 bucket, and report statistics on the contents. -### Pageserver preparations +``` +env SSO_ACCOUNT_ID=123456 REGION=eu-west-1 BUCKET=my-dev-bucket CLOUD_ADMIN_API_TOKEN=${NEON_CLOUD_ADMIN_API_STAGING_KEY} CLOUD_ADMIN_API_URL=[url] cargo run --release -- scan-metadata + +Timelines: 31106 +With errors: 3 +With warnings: 13942 +With garbage: 0 +Index versions: 2: 13942, 4: 17162 +Timeline size bytes: min 22413312, 1% 52133887, 10% 56459263, 50% 101711871, 90% 191561727, 99% 280887295, max 167535558656 +Layer size bytes: min 24576, 1% 36879, 10% 36879, 50% 61471, 90% 44695551, 99% 201457663, max 275324928 +Timeline layer count: min 1, 1% 3, 10% 6, 50% 16, 90% 25, 99% 39, max 1053 +``` + +## Cleaning up running pageservers If S3 state is altered first manually, pageserver in-memory state will contain wrong data about S3 state, and tenants/timelines may get recreated on S3 (due to any layer upload due to compaction, pageserver restart, etc.). So before proceeding, for tenants/timelines which are already deleted in the console, we must remove these from pageservers. diff --git a/s3_scrubber/src/checks.rs b/s3_scrubber/src/checks.rs index 914681d707..c11f1f9779 100644 --- a/s3_scrubber/src/checks.rs +++ b/s3_scrubber/src/checks.rs @@ -1,178 +1,27 @@ -use std::collections::{hash_map, HashMap, HashSet}; -use std::sync::Arc; -use std::time::Duration; +use std::collections::HashSet; use anyhow::Context; use aws_sdk_s3::Client; -use tokio::task::JoinSet; -use tracing::{error, info, info_span, warn, Instrument}; +use tracing::{error, info, warn}; -use crate::cloud_admin_api::{BranchData, CloudAdminApiClient, ProjectId}; -use crate::delete_batch_producer::DeleteProducerStats; -use crate::{download_object_with_retries, list_objects_with_retries, RootTarget, MAX_RETRIES}; +use crate::cloud_admin_api::BranchData; +use crate::{download_object_with_retries, list_objects_with_retries, RootTarget}; use pageserver::tenant::storage_layer::LayerFileName; use pageserver::tenant::IndexPart; use utils::id::TenantTimelineId; -pub async fn validate_pageserver_active_tenant_and_timelines( - s3_client: Arc, - s3_root: RootTarget, - admin_client: Arc, - batch_producer_stats: DeleteProducerStats, -) -> anyhow::Result { - let Some(timeline_stats) = batch_producer_stats.timeline_stats else { - info!("No tenant-only checks, exiting"); - return Ok(BranchCheckStats::default()); - }; - - let s3_active_projects = batch_producer_stats - .tenant_stats - .active_entries - .into_iter() - .map(|project| (project.id.clone(), project)) - .collect::>(); - info!("Validating {} active tenants", s3_active_projects.len()); - - let mut s3_active_branches_per_project = HashMap::>::new(); - let mut s3_blob_data = HashMap::::new(); - for active_branch in timeline_stats.active_entries { - let active_project_id = active_branch.project_id.clone(); - let active_branch_id = active_branch.id.clone(); - let active_timeline_id = active_branch.timeline_id; - - s3_active_branches_per_project - .entry(active_project_id.clone()) - .or_default() - .push(active_branch); - - let Some(active_project) = s3_active_projects.get(&active_project_id) else { - error!( - "Branch {:?} for project {:?} has no such project in the active projects", - active_branch_id, active_project_id - ); - continue; - }; - - let id = TenantTimelineId::new(active_project.tenant, active_timeline_id); - s3_blob_data.insert( - id, - list_timeline_blobs(&s3_client, id, &s3_root) - .await - .with_context(|| format!("List timeline {id} blobs"))?, - ); - } - - let mut branch_checks = JoinSet::new(); - for (_, s3_active_project) in s3_active_projects { - let project_id = &s3_active_project.id; - let tenant_id = s3_active_project.tenant; - - let mut console_active_branches = - branches_for_project_with_retries(&admin_client, project_id) - .await - .with_context(|| { - format!("Client API branches for project {project_id:?} retrieval") - })? - .into_iter() - .map(|branch| (branch.id.clone(), branch)) - .collect::>(); - - let active_branches = s3_active_branches_per_project - .remove(project_id) - .unwrap_or_default(); - info!( - "Spawning tasks for {} tenant {} active timelines", - active_branches.len(), - tenant_id - ); - for s3_active_branch in active_branches { - let console_branch = console_active_branches.remove(&s3_active_branch.id); - let timeline_id = s3_active_branch.timeline_id; - let id = TenantTimelineId::new(tenant_id, timeline_id); - let s3_data = s3_blob_data.remove(&id); - let s3_root = s3_root.clone(); - branch_checks.spawn( - async move { - let check_errors = branch_cleanup_and_check_errors( - &id, - &s3_root, - Some(&s3_active_branch), - console_branch, - s3_data, - ) - .await; - (id, check_errors) - } - .instrument(info_span!("check_timeline", id = %id)), - ); - } - } - - let mut total_stats = BranchCheckStats::default(); - while let Some((id, analysis)) = branch_checks - .join_next() - .await - .transpose() - .context("branch check task join")? - { - total_stats.add(id, analysis.errors); - } - Ok(total_stats) -} - -async fn branches_for_project_with_retries( - admin_client: &CloudAdminApiClient, - project_id: &ProjectId, -) -> anyhow::Result> { - for _ in 0..MAX_RETRIES { - match admin_client.branches_for_project(project_id, false).await { - Ok(branches) => return Ok(branches), - Err(e) => { - error!("admin list branches for project {project_id:?} query failed: {e}"); - tokio::time::sleep(Duration::from_secs(1)).await; - } - } - } - - anyhow::bail!("Failed to list branches for project {project_id:?} {MAX_RETRIES} times") -} - -#[derive(Debug, Default)] -pub struct BranchCheckStats { - pub timelines_with_errors: HashMap>, - pub normal_timelines: HashSet, -} - -impl BranchCheckStats { - pub fn add(&mut self, id: TenantTimelineId, check_errors: Vec) { - if check_errors.is_empty() { - if !self.normal_timelines.insert(id) { - panic!("Checking branch with timeline {id} more than once") - } - } else { - match self.timelines_with_errors.entry(id) { - hash_map::Entry::Occupied(_) => { - panic!("Checking branch with timeline {id} more than once") - } - hash_map::Entry::Vacant(v) => { - v.insert(check_errors); - } - } - } - } -} - -pub struct TimelineAnalysis { +pub(crate) struct TimelineAnalysis { /// Anomalies detected - pub errors: Vec, + pub(crate) errors: Vec, /// Healthy-but-noteworthy, like old-versioned structures that are readable but /// worth reporting for awareness that we must not remove that old version decoding /// yet. - pub warnings: Vec, + pub(crate) warnings: Vec, - /// Keys not referenced in metadata: candidates for removal - pub garbage_keys: Vec, + /// Keys not referenced in metadata: candidates for removal, but NOT NECESSARILY: beware + /// of races between reading the metadata and reading the objects. + pub(crate) garbage_keys: Vec, } impl TimelineAnalysis { @@ -185,7 +34,7 @@ impl TimelineAnalysis { } } -pub async fn branch_cleanup_and_check_errors( +pub(crate) async fn branch_cleanup_and_check_errors( id: &TenantTimelineId, s3_root: &RootTarget, s3_active_branch: Option<&BranchData>, @@ -320,13 +169,13 @@ pub async fn branch_cleanup_and_check_errors( } #[derive(Debug)] -pub struct S3TimelineBlobData { - pub blob_data: BlobDataParseResult, - pub keys_to_remove: Vec, +pub(crate) struct S3TimelineBlobData { + pub(crate) blob_data: BlobDataParseResult, + pub(crate) keys_to_remove: Vec, } #[derive(Debug)] -pub enum BlobDataParseResult { +pub(crate) enum BlobDataParseResult { Parsed { index_part: IndexPart, s3_layers: HashSet, @@ -334,7 +183,7 @@ pub enum BlobDataParseResult { Incorrect(Vec), } -pub async fn list_timeline_blobs( +pub(crate) async fn list_timeline_blobs( s3_client: &Client, id: TenantTimelineId, s3_root: &RootTarget, diff --git a/s3_scrubber/src/cloud_admin_api.rs b/s3_scrubber/src/cloud_admin_api.rs index 3c21b70385..1b28dc4dff 100644 --- a/s3_scrubber/src/cloud_admin_api.rs +++ b/s3_scrubber/src/cloud_admin_api.rs @@ -1,12 +1,19 @@ #![allow(unused)] +use std::str::FromStr; +use std::time::Duration; + use chrono::{DateTime, Utc}; -use reqwest::{header, Client, Url}; +use hex::FromHex; +use reqwest::{header, Client, StatusCode, Url}; +use serde::Deserialize; use tokio::sync::Semaphore; use utils::id::{TenantId, TimelineId}; use utils::lsn::Lsn; +use crate::ConsoleConfig; + #[derive(Debug)] pub struct Error { context: String, @@ -34,6 +41,9 @@ impl std::fmt::Display for Error { self.context, e ) } + ErrorKind::ResponseStatus(status) => { + write!(f, "Bad response status {}: {}", status, self.context) + } ErrorKind::UnexpectedState => write!(f, "Unexpected state: {}", self.context), } } @@ -53,6 +63,7 @@ impl std::error::Error for Error {} pub enum ErrorKind { RequestSend(reqwest::Error), BodyRead(reqwest::Error), + ResponseStatus(StatusCode), UnexpectedState, } @@ -100,7 +111,23 @@ pub struct SafekeeperData { pub availability_zone_id: String, } -#[serde_with::serde_as] +/// For ID fields, the Console API does not always return a value or null. It will +/// sometimes return an empty string. Our native Id type does not consider this acceptable +/// (nor should it), so we use a wrapper for talking to the Console API. +fn from_nullable_id<'de, D>(deserializer: D) -> Result +where + D: serde::de::Deserializer<'de>, +{ + let id_str = String::deserialize(deserializer)?; + if id_str.is_empty() { + // This is a bogus value, but for the purposes of the scrubber all that + // matters is that it doesn't collide with any real IDs. + Ok(TenantId::from([0u8; 16])) + } else { + TenantId::from_hex(&id_str).map_err(|e| serde::de::Error::custom(format!("{e}"))) + } +} + #[derive(Debug, Clone, serde::Deserialize)] pub struct ProjectData { pub id: ProjectId, @@ -109,7 +136,7 @@ pub struct ProjectData { pub platform_id: String, pub user_id: String, pub pageserver_id: u64, - #[serde_as(as = "serde_with::DisplayFromStr")] + #[serde(deserialize_with = "from_nullable_id")] pub tenant: TenantId, pub safekeepers: Vec, pub deleted: bool, @@ -148,11 +175,27 @@ pub struct BranchData { pub written_size: Option, } +pub trait MaybeDeleted { + fn is_deleted(&self) -> bool; +} + +impl MaybeDeleted for ProjectData { + fn is_deleted(&self) -> bool { + self.deleted + } +} + +impl MaybeDeleted for BranchData { + fn is_deleted(&self) -> bool { + self.deleted + } +} + impl CloudAdminApiClient { - pub fn new(token: String, base_url: Url) -> Self { + pub fn new(config: ConsoleConfig) -> Self { Self { - token, - base_url, + token: config.token, + base_url: config.base_url, request_limiter: Semaphore::new(200), http_client: Client::new(), // TODO timeout configs at least } @@ -208,6 +251,81 @@ impl CloudAdminApiClient { } } + pub async fn list_projects(&self, region_id: String) -> Result, Error> { + let _permit = self + .request_limiter + .acquire() + .await + .expect("Semaphore is not closed"); + + let mut pagination_offset = 0; + const PAGINATION_LIMIT: usize = 512; + let mut result: Vec = Vec::with_capacity(PAGINATION_LIMIT); + loop { + let response = self + .http_client + .get(self.append_url("/projects")) + .query(&[ + ("show_deleted", "false".to_string()), + ("limit", format!("{PAGINATION_LIMIT}")), + ("offset", format!("{pagination_offset}")), + ]) + .header(header::ACCEPT, "application/json") + .bearer_auth(&self.token) + .send() + .await + .map_err(|e| { + Error::new( + "List active projects".to_string(), + ErrorKind::RequestSend(e), + ) + })?; + + match response.status() { + StatusCode::OK => {} + StatusCode::SERVICE_UNAVAILABLE | StatusCode::TOO_MANY_REQUESTS => { + tokio::time::sleep(Duration::from_millis(500)).await; + continue; + } + status => { + return Err(Error::new( + "List active projects".to_string(), + ErrorKind::ResponseStatus(response.status()), + )) + } + } + + let response_bytes = response.bytes().await.map_err(|e| { + Error::new("List active projects".to_string(), ErrorKind::BodyRead(e)) + })?; + + let decode_result = + serde_json::from_slice::>>(&response_bytes); + + let mut response = match decode_result { + Ok(r) => r, + Err(decode) => { + tracing::error!( + "Failed to decode response body: {}\n{}", + decode, + String::from_utf8(response_bytes.to_vec()).unwrap() + ); + panic!("we out"); + } + }; + + pagination_offset += response.data.len(); + + result.extend(response.data.drain(..).filter(|t| t.region_id == region_id)); + + if pagination_offset >= response.total.unwrap_or(0) { + break; + } + } + + Ok(result) + } + pub async fn find_timeline_branch( &self, timeline_id: TimelineId, diff --git a/s3_scrubber/src/delete_batch_producer.rs b/s3_scrubber/src/delete_batch_producer.rs deleted file mode 100644 index 99ab5c4198..0000000000 --- a/s3_scrubber/src/delete_batch_producer.rs +++ /dev/null @@ -1,354 +0,0 @@ -mod tenant_batch; -mod timeline_batch; - -use std::future::Future; -use std::str::FromStr; -use std::sync::Arc; -use std::time::Duration; - -use anyhow::Context; -use aws_sdk_s3::Client; -use either::Either; -use tokio::sync::mpsc::UnboundedReceiver; -use tokio::sync::Mutex; -use tokio::task::{JoinHandle, JoinSet}; -use tracing::{error, info, info_span, Instrument}; - -use crate::cloud_admin_api::{BranchData, CloudAdminApiClient, ProjectData}; -use crate::{list_objects_with_retries, RootTarget, S3Target, TraversingDepth, MAX_RETRIES}; -use utils::id::{TenantId, TenantTimelineId}; - -/// Typical tenant to remove contains 1 layer and 1 index_part.json blobs -/// Also, there are some non-standard tenants to remove, having more layers. -/// delete_objects request allows up to 1000 keys, so be on a safe side and allow most -/// batch processing tasks to do 1 delete objects request only. -/// -/// Every batch item will be additionally S3 LS'ed later, so keep the batch size -/// even lower to allow multiple concurrent tasks do the LS requests. -const BATCH_SIZE: usize = 100; - -pub struct DeleteBatchProducer { - delete_tenants_sender_task: JoinHandle>>, - delete_timelines_sender_task: - JoinHandle>>, - delete_batch_creator_task: JoinHandle<()>, - delete_batch_receiver: Arc>>, -} - -pub struct DeleteProducerStats { - pub tenant_stats: ProcessedS3List, - pub timeline_stats: Option>, -} - -impl DeleteProducerStats { - pub fn tenants_checked(&self) -> usize { - self.tenant_stats.entries_total - } - - pub fn active_tenants(&self) -> usize { - self.tenant_stats.active_entries.len() - } - - pub fn timelines_checked(&self) -> usize { - self.timeline_stats - .as_ref() - .map(|stats| stats.entries_total) - .unwrap_or(0) - } -} - -#[derive(Debug, Default, Clone)] -pub struct DeleteBatch { - pub tenants: Vec, - pub timelines: Vec, -} - -impl DeleteBatch { - pub fn merge(&mut self, other: Self) { - self.tenants.extend(other.tenants); - self.timelines.extend(other.timelines); - } - - pub fn len(&self) -> usize { - self.tenants.len() + self.timelines.len() - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } -} - -impl DeleteBatchProducer { - pub fn start( - admin_client: Arc, - s3_client: Arc, - s3_root_target: RootTarget, - traversing_depth: TraversingDepth, - ) -> Self { - let (delete_elements_sender, mut delete_elements_receiver) = - tokio::sync::mpsc::unbounded_channel(); - let delete_elements_sender = Arc::new(delete_elements_sender); - let admin_client = Arc::new(admin_client); - - let (projects_to_check_sender, mut projects_to_check_receiver) = - tokio::sync::mpsc::unbounded_channel(); - let delete_tenants_root_target = s3_root_target.clone(); - let delete_tenants_client = Arc::clone(&s3_client); - let delete_tenants_admin_client = Arc::clone(&admin_client); - let delete_sender = Arc::clone(&delete_elements_sender); - let delete_tenants_sender_task = tokio::spawn( - async move { - tenant_batch::schedule_cleanup_deleted_tenants( - &delete_tenants_root_target, - &delete_tenants_client, - &delete_tenants_admin_client, - projects_to_check_sender, - delete_sender, - traversing_depth, - ) - .await - } - .instrument(info_span!("delete_tenants_sender")), - ); - let delete_timelines_sender_task = tokio::spawn(async move { - timeline_batch::schedule_cleanup_deleted_timelines( - &s3_root_target, - &s3_client, - &admin_client, - &mut projects_to_check_receiver, - delete_elements_sender, - ) - .in_current_span() - .await - }); - - let (delete_batch_sender, delete_batch_receiver) = tokio::sync::mpsc::unbounded_channel(); - let delete_batch_creator_task = tokio::spawn( - async move { - 'outer: loop { - let mut delete_batch = DeleteBatch::default(); - while delete_batch.len() < BATCH_SIZE { - match delete_elements_receiver.recv().await { - Some(new_task) => match new_task { - Either::Left(tenant_id) => delete_batch.tenants.push(tenant_id), - Either::Right(timeline_id) => { - delete_batch.timelines.push(timeline_id) - } - }, - None => { - info!("Task finished: sender dropped"); - delete_batch_sender.send(delete_batch).ok(); - break 'outer; - } - } - } - - if !delete_batch.is_empty() { - delete_batch_sender.send(delete_batch).ok(); - } - } - } - .instrument(info_span!("delete batch creator")), - ); - - Self { - delete_tenants_sender_task, - delete_timelines_sender_task, - delete_batch_creator_task, - delete_batch_receiver: Arc::new(Mutex::new(delete_batch_receiver)), - } - } - - pub fn subscribe(&self) -> Arc>> { - self.delete_batch_receiver.clone() - } - - pub async fn join(self) -> anyhow::Result { - let (delete_tenants_task_result, delete_timelines_task_result, batch_task_result) = tokio::join!( - self.delete_tenants_sender_task, - self.delete_timelines_sender_task, - self.delete_batch_creator_task, - ); - - let tenant_stats = match delete_tenants_task_result { - Ok(Ok(stats)) => stats, - Ok(Err(tenant_deletion_error)) => return Err(tenant_deletion_error), - Err(join_error) => { - anyhow::bail!("Failed to join the delete tenant producing task: {join_error}") - } - }; - - let timeline_stats = match delete_timelines_task_result { - Ok(Ok(stats)) => Some(stats), - Ok(Err(timeline_deletion_error)) => return Err(timeline_deletion_error), - Err(join_error) => { - anyhow::bail!("Failed to join the delete timeline producing task: {join_error}") - } - }; - - match batch_task_result { - Ok(()) => (), - Err(join_error) => anyhow::bail!("Failed to join the batch forming task: {join_error}"), - }; - - Ok(DeleteProducerStats { - tenant_stats, - timeline_stats, - }) - } -} - -pub struct ProcessedS3List { - pub entries_total: usize, - pub entries_to_delete: Vec, - pub active_entries: Vec, -} - -impl Default for ProcessedS3List { - fn default() -> Self { - Self { - entries_total: 0, - entries_to_delete: Vec::new(), - active_entries: Vec::new(), - } - } -} - -impl ProcessedS3List { - fn merge(&mut self, other: Self) { - self.entries_total += other.entries_total; - self.entries_to_delete.extend(other.entries_to_delete); - self.active_entries.extend(other.active_entries); - } - - fn change_ids(self, transform: impl Fn(I) -> NewI) -> ProcessedS3List { - ProcessedS3List { - entries_total: self.entries_total, - entries_to_delete: self.entries_to_delete.into_iter().map(transform).collect(), - active_entries: self.active_entries, - } - } -} - -async fn process_s3_target_recursively( - s3_client: &Client, - target: &S3Target, - find_active_and_deleted_entries: F, -) -> anyhow::Result> -where - I: FromStr + Send + Sync, - E: Send + Sync + std::error::Error + 'static, - F: FnOnce(Vec) -> Fut + Clone, - Fut: Future>>, -{ - let mut continuation_token = None; - let mut total_entries = ProcessedS3List::default(); - - loop { - let fetch_response = - list_objects_with_retries(s3_client, target, continuation_token.clone()).await?; - - let new_entry_ids = fetch_response - .common_prefixes() - .unwrap_or_default() - .iter() - .filter_map(|prefix| prefix.prefix()) - .filter_map(|prefix| -> Option<&str> { - prefix - .strip_prefix(&target.prefix_in_bucket)? - .strip_suffix('/') - }) - .map(|entry_id_str| { - entry_id_str - .parse() - .with_context(|| format!("Incorrect entry id str: {entry_id_str}")) - }) - .collect::>>() - .context("list and parse bucket's entry ids")?; - - total_entries.merge( - (find_active_and_deleted_entries.clone())(new_entry_ids) - .await - .context("filter active and deleted entry ids")?, - ); - - match fetch_response.next_continuation_token { - Some(new_token) => continuation_token = Some(new_token), - None => break, - } - } - - Ok(total_entries) -} - -enum FetchResult { - Found(A), - Deleted, - Absent, -} - -async fn split_to_active_and_deleted_entries( - new_entry_ids: Vec, - find_active_entry: F, -) -> anyhow::Result> -where - I: std::fmt::Display + Send + Sync + 'static + Copy, - A: Send + 'static, - F: FnOnce(I) -> Fut + Send + Sync + 'static + Clone, - Fut: Future>> + Send, -{ - let entries_total = new_entry_ids.len(); - let mut check_tasks = JoinSet::new(); - let mut active_entries = Vec::with_capacity(entries_total); - let mut entries_to_delete = Vec::with_capacity(entries_total); - - for new_entry_id in new_entry_ids { - let check_closure = find_active_entry.clone(); - check_tasks.spawn( - async move { - ( - new_entry_id, - async { - for _ in 0..MAX_RETRIES { - let closure_clone = check_closure.clone(); - match closure_clone(new_entry_id).await { - Ok(active_entry) => return Ok(active_entry), - Err(e) => { - error!("find active entry admin API call failed: {e}"); - tokio::time::sleep(Duration::from_secs(1)).await; - } - } - } - - anyhow::bail!("Failed to check entry {new_entry_id} {MAX_RETRIES} times") - } - .await, - ) - } - .instrument(info_span!("filter_active_entries")), - ); - } - - while let Some(task_result) = check_tasks.join_next().await { - let (entry_id, entry_data_fetch_result) = task_result.context("task join")?; - match entry_data_fetch_result.context("entry data fetch")? { - FetchResult::Found(active_entry) => { - info!("Entry {entry_id} is alive, cannot delete"); - active_entries.push(active_entry); - } - FetchResult::Deleted => { - info!("Entry {entry_id} deleted in the admin data, can safely delete"); - entries_to_delete.push(entry_id); - } - FetchResult::Absent => { - info!("Entry {entry_id} absent in the admin data, can safely delete"); - entries_to_delete.push(entry_id); - } - } - } - Ok(ProcessedS3List { - entries_total, - entries_to_delete, - active_entries, - }) -} diff --git a/s3_scrubber/src/delete_batch_producer/tenant_batch.rs b/s3_scrubber/src/delete_batch_producer/tenant_batch.rs deleted file mode 100644 index 59fd638645..0000000000 --- a/s3_scrubber/src/delete_batch_producer/tenant_batch.rs +++ /dev/null @@ -1,87 +0,0 @@ -use std::sync::Arc; - -use anyhow::Context; -use aws_sdk_s3::Client; -use either::Either; -use tokio::sync::mpsc::UnboundedSender; -use tracing::info; - -use crate::cloud_admin_api::{CloudAdminApiClient, ProjectData}; -use crate::delete_batch_producer::FetchResult; -use crate::{RootTarget, TraversingDepth}; -use utils::id::{TenantId, TenantTimelineId}; - -use super::ProcessedS3List; - -pub async fn schedule_cleanup_deleted_tenants( - s3_root_target: &RootTarget, - s3_client: &Arc, - admin_client: &Arc, - projects_to_check_sender: UnboundedSender, - delete_sender: Arc>>, - traversing_depth: TraversingDepth, -) -> anyhow::Result> { - info!( - "Starting to list the bucket from root {}", - s3_root_target.bucket_name() - ); - s3_client - .head_bucket() - .bucket(s3_root_target.bucket_name()) - .send() - .await - .with_context(|| format!("bucket {} was not found", s3_root_target.bucket_name()))?; - - let check_client = Arc::clone(admin_client); - let tenant_stats = super::process_s3_target_recursively( - s3_client, - s3_root_target.tenants_root(), - |s3_tenants| async move { - let another_client = Arc::clone(&check_client); - super::split_to_active_and_deleted_entries(s3_tenants, move |tenant_id| async move { - let project_data = another_client - .find_tenant_project(tenant_id) - .await - .with_context(|| format!("Tenant {tenant_id} project admin check"))?; - - Ok(if let Some(console_project) = project_data { - if console_project.deleted { - delete_sender.send(Either::Left(tenant_id)).ok(); - FetchResult::Deleted - } else { - if traversing_depth == TraversingDepth::Timeline { - projects_to_check_sender.send(console_project.clone()).ok(); - } - FetchResult::Found(console_project) - } - } else { - delete_sender.send(Either::Left(tenant_id)).ok(); - FetchResult::Absent - }) - }) - .await - }, - ) - .await - .context("tenant batch processing")?; - - info!( - "Among {} tenants, found {} tenants to delete and {} active ones", - tenant_stats.entries_total, - tenant_stats.entries_to_delete.len(), - tenant_stats.active_entries.len(), - ); - - let tenant_stats = match traversing_depth { - TraversingDepth::Tenant => { - info!("Finished listing the bucket for tenants only"); - tenant_stats - } - TraversingDepth::Timeline => { - info!("Finished listing the bucket for tenants and sent {} active tenants to check for timelines", tenant_stats.active_entries.len()); - tenant_stats - } - }; - - Ok(tenant_stats) -} diff --git a/s3_scrubber/src/delete_batch_producer/timeline_batch.rs b/s3_scrubber/src/delete_batch_producer/timeline_batch.rs deleted file mode 100644 index 2ad522d3fb..0000000000 --- a/s3_scrubber/src/delete_batch_producer/timeline_batch.rs +++ /dev/null @@ -1,102 +0,0 @@ -use std::sync::Arc; - -use anyhow::Context; -use aws_sdk_s3::Client; -use either::Either; -use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; -use tracing::{info, info_span, Instrument}; - -use crate::cloud_admin_api::{BranchData, CloudAdminApiClient, ProjectData}; -use crate::delete_batch_producer::{FetchResult, ProcessedS3List}; -use crate::RootTarget; -use utils::id::{TenantId, TenantTimelineId}; - -pub async fn schedule_cleanup_deleted_timelines( - s3_root_target: &RootTarget, - s3_client: &Arc, - admin_client: &Arc, - projects_to_check_receiver: &mut UnboundedReceiver, - delete_elements_sender: Arc>>, -) -> anyhow::Result> { - info!( - "Starting to list the bucket from root {}", - s3_root_target.bucket_name() - ); - s3_client - .head_bucket() - .bucket(s3_root_target.bucket_name()) - .send() - .await - .with_context(|| format!("bucket {} was not found", s3_root_target.bucket_name()))?; - - let mut timeline_stats = ProcessedS3List::default(); - while let Some(project_to_check) = projects_to_check_receiver.recv().await { - let check_client = Arc::clone(admin_client); - - let check_s3_client = Arc::clone(s3_client); - - let check_delete_sender = Arc::clone(&delete_elements_sender); - - let check_root = s3_root_target.clone(); - - let new_stats = async move { - let tenant_id_to_check = project_to_check.tenant; - let check_target = check_root.timelines_root(&tenant_id_to_check); - let stats = super::process_s3_target_recursively( - &check_s3_client, - &check_target, - |s3_timelines| async move { - let another_client = check_client.clone(); - super::split_to_active_and_deleted_entries( - s3_timelines, - move |timeline_id| async move { - let console_branch = another_client - .find_timeline_branch(timeline_id) - .await - .map_err(|e| { - anyhow::anyhow!( - "Timeline {timeline_id} branch admin check: {e}" - ) - })?; - - let id = TenantTimelineId::new(tenant_id_to_check, timeline_id); - Ok(match console_branch { - Some(console_branch) => { - if console_branch.deleted { - check_delete_sender.send(Either::Right(id)).ok(); - FetchResult::Deleted - } else { - FetchResult::Found(console_branch) - } - } - None => { - check_delete_sender.send(Either::Right(id)).ok(); - FetchResult::Absent - } - }) - }, - ) - .await - }, - ) - .await - .with_context(|| format!("tenant {tenant_id_to_check} timeline batch processing"))? - .change_ids(|timeline_id| TenantTimelineId::new(tenant_id_to_check, timeline_id)); - - Ok::<_, anyhow::Error>(stats) - } - .instrument(info_span!("delete_timelines_sender", tenant = %project_to_check.tenant)) - .await?; - - timeline_stats.merge(new_stats); - } - - info!( - "Among {} timelines, found {} timelines to delete and {} active ones", - timeline_stats.entries_total, - timeline_stats.entries_to_delete.len(), - timeline_stats.active_entries.len(), - ); - - Ok(timeline_stats) -} diff --git a/s3_scrubber/src/garbage.rs b/s3_scrubber/src/garbage.rs new file mode 100644 index 0000000000..daeb5e9777 --- /dev/null +++ b/s3_scrubber/src/garbage.rs @@ -0,0 +1,419 @@ +//! Functionality for finding and purging garbage, as in "garbage collection". Garbage means +//! S3 objects which are either not referenced by any metadata, or are referenced by a +//! control plane tenant/timeline in a deleted state. + +use std::{collections::HashMap, sync::Arc}; + +use anyhow::Context; +use aws_sdk_s3::{ + types::{Delete, ObjectIdentifier}, + Client, +}; +use futures_util::{pin_mut, TryStreamExt}; +use serde::{Deserialize, Serialize}; +use tokio_stream::StreamExt; +use utils::id::{TenantId, TenantTimelineId}; + +use crate::{ + cloud_admin_api::{CloudAdminApiClient, MaybeDeleted, ProjectData}, + init_remote, + metadata_stream::{stream_listing, stream_tenant_timelines, stream_tenants}, + BucketConfig, ConsoleConfig, NodeKind, RootTarget, TraversingDepth, +}; + +#[derive(Serialize, Deserialize, Debug)] +enum GarbageReason { + DeletedInConsole, + MissingInConsole, +} + +#[derive(Serialize, Deserialize, Debug)] +enum GarbageEntity { + Tenant(TenantId), + Timeline(TenantTimelineId), +} + +#[derive(Serialize, Deserialize, Debug)] +struct GarbageItem { + entity: GarbageEntity, + reason: GarbageReason, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct GarbageList { + /// Remember what NodeKind we were finding garbage for, so that we can + /// purge the list without re-stating it. + node_kind: NodeKind, + + /// Embed the identity of the bucket, so that we do not risk executing + /// the wrong list against the wrong bucket, and so that the user does not have + /// to re-state the bucket details when purging. + bucket_config: BucketConfig, + + items: Vec, + + /// Advisory information to enable consumers to do a validation that if we + /// see garbage, we saw some active tenants too. This protects against classes of bugs + /// in the scrubber that might otherwise generate a "deleted all" result. + active_tenant_count: usize, +} + +impl GarbageList { + fn new(node_kind: NodeKind, bucket_config: BucketConfig) -> Self { + Self { + items: Vec::new(), + active_tenant_count: 0, + node_kind, + bucket_config, + } + } + + /// Return true if appended, false if not. False means the result was not garbage. + fn maybe_append(&mut self, entity: GarbageEntity, result: Option) -> bool + where + T: MaybeDeleted, + { + match result { + Some(result_item) if result_item.is_deleted() => { + self.items.push(GarbageItem { + entity, + reason: GarbageReason::DeletedInConsole, + }); + true + } + Some(_) => false, + None => { + self.items.push(GarbageItem { + entity, + reason: GarbageReason::MissingInConsole, + }); + true + } + } + } +} + +pub async fn find_garbage( + bucket_config: BucketConfig, + console_config: ConsoleConfig, + depth: TraversingDepth, + node_kind: NodeKind, + output_path: String, +) -> anyhow::Result<()> { + let garbage = find_garbage_inner(bucket_config, console_config, depth, node_kind).await?; + let serialized = serde_json::to_vec_pretty(&garbage)?; + + tokio::fs::write(&output_path, &serialized).await?; + + tracing::info!("Wrote garbage report to {output_path}"); + + Ok(()) +} + +// How many concurrent S3 operations to issue (approximately): this is the concurrency +// for things like listing the timelines within tenant prefixes. +const S3_CONCURRENCY: usize = 32; + +// How many concurrent API requests to make to the console API. +const CONSOLE_CONCURRENCY: usize = 128; + +async fn find_garbage_inner( + bucket_config: BucketConfig, + console_config: ConsoleConfig, + depth: TraversingDepth, + node_kind: NodeKind, +) -> anyhow::Result { + // Construct clients for S3 and for Console API + let (s3_client, target) = init_remote(bucket_config.clone(), node_kind)?; + let cloud_admin_api_client = Arc::new(CloudAdminApiClient::new(console_config)); + + // Build a set of console-known tenants, for quickly eliminating known-active tenants without having + // to issue O(N) console API requests. + let console_projects: HashMap = cloud_admin_api_client + // FIXME: we can't just assume that all console's region ids are aws-. This hack + // will go away when we are talking to Control Plane APIs, which are per-region. + .list_projects(format!("aws-{}", bucket_config.region)) + .await? + .into_iter() + .map(|t| (t.tenant, t)) + .collect(); + tracing::info!( + "Loaded {} console projects tenant IDs", + console_projects.len() + ); + + // Enumerate Tenants in S3, and check if each one exists in Console + tracing::info!("Finding all tenants in bucket {}...", bucket_config.bucket); + let tenants = stream_tenants(&s3_client, &target); + let tenants_checked = tenants.map_ok(|t| { + let api_client = cloud_admin_api_client.clone(); + let console_projects = &console_projects; + async move { + match console_projects.get(&t) { + Some(project_data) => Ok((t, Some(project_data.clone()))), + None => api_client + .find_tenant_project(t) + .await + .map_err(|e| anyhow::anyhow!(e)) + .map(|r| (t, r)), + } + } + }); + let tenants_checked = tenants_checked.try_buffer_unordered(CONSOLE_CONCURRENCY); + + // Process the results of Tenant checks. If a Tenant is garbage, it goes into + // the `GarbageList`. Else it goes into `active_tenants` for more detailed timeline + // checks if they are enabled by the `depth` parameter. + pin_mut!(tenants_checked); + let mut garbage = GarbageList::new(node_kind, bucket_config); + let mut active_tenants: Vec = vec![]; + let mut counter = 0; + while let Some(result) = tenants_checked.next().await { + let (tenant_id, console_result) = result?; + + // Paranoia check + if let Some(project) = &console_result { + assert!(project.tenant == tenant_id); + } + + if garbage.maybe_append(GarbageEntity::Tenant(tenant_id), console_result) { + tracing::debug!("Tenant {tenant_id} is garbage"); + } else { + tracing::debug!("Tenant {tenant_id} is active"); + active_tenants.push(tenant_id); + } + + counter += 1; + if counter % 1000 == 0 { + tracing::info!( + "Progress: {counter} tenants checked, {} active, {} garbage", + active_tenants.len(), + garbage.items.len() + ); + } + } + + tracing::info!( + "Found {}/{} garbage tenants", + garbage.items.len(), + garbage.items.len() + active_tenants.len() + ); + + // If we are only checking tenant-deep, we are done. Otherwise we must + // proceed to check the individual timelines of the active tenants. + if depth == TraversingDepth::Tenant { + return Ok(garbage); + } + + tracing::info!( + "Checking timelines for {} active tenants", + active_tenants.len(), + ); + + // Construct a stream of all timelines within active tenants + let active_tenants = tokio_stream::iter(active_tenants.iter().map(Ok)); + let timelines = active_tenants.map_ok(|t| stream_tenant_timelines(&s3_client, &target, *t)); + let timelines = timelines.try_buffer_unordered(S3_CONCURRENCY); + let timelines = timelines.try_flatten(); + + // For all timelines within active tenants, call into console API to check their existence + let timelines_checked = timelines.map_ok(|ttid| { + let api_client = cloud_admin_api_client.clone(); + async move { + api_client + .find_timeline_branch(ttid.timeline_id) + .await + .map_err(|e| anyhow::anyhow!(e)) + .map(|r| (ttid, r)) + } + }); + let timelines_checked = timelines_checked.try_buffer_unordered(CONSOLE_CONCURRENCY); + + // Update the GarbageList with any timelines which appear not to exist. + pin_mut!(timelines_checked); + while let Some(result) = timelines_checked.next().await { + let (ttid, console_result) = result?; + if garbage.maybe_append(GarbageEntity::Timeline(ttid), console_result) { + tracing::debug!("Timeline {ttid} is garbage"); + } else { + tracing::debug!("Timeline {ttid} is active"); + } + } + + Ok(garbage) +} + +#[derive(clap::ValueEnum, Debug, Clone)] +pub enum PurgeMode { + /// The safest mode: only delete tenants that were explicitly reported as deleted + /// by Console API. + DeletedOnly, + + /// Delete all garbage tenants, including those which are only presumed to be deleted, + /// because the Console API could not find them. + DeletedAndMissing, +} + +impl std::fmt::Display for PurgeMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + PurgeMode::DeletedOnly => write!(f, "deleted-only"), + PurgeMode::DeletedAndMissing => write!(f, "deleted-and-missing"), + } + } +} + +pub async fn get_tenant_objects( + s3_client: &Arc, + target: RootTarget, + tenant_id: TenantId, +) -> anyhow::Result> { + tracing::debug!("Listing objects in tenant {tenant_id}"); + // TODO: apply extra validation based on object modification time. Don't purge + // tenants where any timeline's index_part.json has been touched recently. + + let mut tenant_root = target.tenant_root(&tenant_id); + + // Remove delimiter, so that object listing lists all keys in the prefix and not just + // common prefixes. + tenant_root.delimiter = String::new(); + + let key_stream = stream_listing(s3_client, &tenant_root); + key_stream.try_collect().await +} + +pub async fn get_timeline_objects( + s3_client: &Arc, + target: RootTarget, + ttid: TenantTimelineId, +) -> anyhow::Result> { + tracing::debug!("Listing objects in timeline {ttid}"); + let mut timeline_root = target.timeline_root(&ttid); + + // TODO: apply extra validation based on object modification time. Don't purge + // timelines whose index_part.json has been touched recently. + + // Remove delimiter, so that object listing lists all keys in the prefix and not just + // common prefixes. + timeline_root.delimiter = String::new(); + let key_stream = stream_listing(s3_client, &timeline_root); + + key_stream.try_collect().await +} + +const MAX_KEYS_PER_DELETE: usize = 1000; + +/// Drain a buffer of keys into DeleteObjects requests +async fn do_delete( + s3_client: &Arc, + bucket_name: &str, + keys: &mut Vec, + dry_run: bool, + drain: bool, +) -> anyhow::Result<()> { + while (!keys.is_empty() && drain) || (keys.len() >= MAX_KEYS_PER_DELETE) { + let request_keys = + keys.split_off(keys.len() - (std::cmp::min(MAX_KEYS_PER_DELETE, keys.len()))); + if dry_run { + tracing::info!("Dry-run deletion of objects: "); + for k in request_keys { + tracing::info!(" {k:?}"); + } + } else { + let delete_request = s3_client + .delete_objects() + .bucket(bucket_name) + .delete(Delete::builder().set_objects(Some(request_keys)).build()); + delete_request + .send() + .await + .context("DeleteObjects request")?; + } + } + + Ok(()) +} + +pub async fn purge_garbage( + input_path: String, + mode: PurgeMode, + dry_run: bool, +) -> anyhow::Result<()> { + let list_bytes = tokio::fs::read(&input_path).await?; + let garbage_list = serde_json::from_slice::(&list_bytes)?; + tracing::info!( + "Loaded {} items in garbage list from {}", + garbage_list.items.len(), + input_path + ); + + let (s3_client, target) = + init_remote(garbage_list.bucket_config.clone(), garbage_list.node_kind)?; + + // Sanity checks on the incoming list + if garbage_list.active_tenant_count == 0 { + anyhow::bail!("Refusing to purge a garbage list that reports 0 active tenants"); + } + + let filtered_items = garbage_list + .items + .iter() + .filter(|i| match (&mode, &i.reason) { + (PurgeMode::DeletedAndMissing, _) => true, + (PurgeMode::DeletedOnly, GarbageReason::DeletedInConsole) => true, + (PurgeMode::DeletedOnly, GarbageReason::MissingInConsole) => false, + }); + + tracing::info!( + "Filtered down to {} garbage items based on mode {}", + garbage_list.items.len(), + mode + ); + + let items = tokio_stream::iter(filtered_items.map(Ok)); + let get_objects_results = items.map_ok(|i| { + let s3_client = s3_client.clone(); + let target = target.clone(); + async move { + match i.entity { + GarbageEntity::Tenant(tenant_id) => { + get_tenant_objects(&s3_client, target, tenant_id).await + } + GarbageEntity::Timeline(ttid) => { + get_timeline_objects(&s3_client, target, ttid).await + } + } + } + }); + let get_objects_results = get_objects_results.try_buffer_unordered(S3_CONCURRENCY); + + pin_mut!(get_objects_results); + let mut objects_to_delete = Vec::new(); + while let Some(result) = get_objects_results.next().await { + let mut object_list = result?; + objects_to_delete.append(&mut object_list); + if objects_to_delete.len() >= MAX_KEYS_PER_DELETE { + do_delete( + &s3_client, + &garbage_list.bucket_config.bucket, + &mut objects_to_delete, + dry_run, + false, + ) + .await?; + } + } + + do_delete( + &s3_client, + &garbage_list.bucket_config.bucket, + &mut objects_to_delete, + dry_run, + true, + ) + .await?; + + tracing::info!("Fell through"); + + Ok(()) +} diff --git a/s3_scrubber/src/lib.rs b/s3_scrubber/src/lib.rs index 072cdf1d93..98e7ed5334 100644 --- a/s3_scrubber/src/lib.rs +++ b/s3_scrubber/src/lib.rs @@ -1,12 +1,12 @@ pub mod checks; pub mod cloud_admin_api; -pub mod delete_batch_producer; +pub mod garbage; pub mod metadata_stream; -mod s3_deletion; pub mod scan_metadata; use std::env; use std::fmt::Display; +use std::sync::Arc; use std::time::Duration; use anyhow::Context; @@ -17,8 +17,10 @@ use aws_config::sso::SsoCredentialsProvider; use aws_sdk_s3::config::Region; use aws_sdk_s3::{Client, Config}; +use clap::ValueEnum; +use pageserver::tenant::TENANTS_SEGMENT_NAME; use reqwest::Url; -pub use s3_deletion::S3Deleter; +use serde::{Deserialize, Serialize}; use std::io::IsTerminal; use tokio::io::AsyncReadExt; use tracing::error; @@ -29,8 +31,6 @@ use utils::id::{TenantId, TenantTimelineId}; const MAX_RETRIES: usize = 20; const CLOUD_ADMIN_API_TOKEN_ENV_VAR: &str = "CLOUD_ADMIN_API_TOKEN"; -pub const CLI_NAME: &str = "s3-scrubber"; - #[derive(Debug, Clone)] pub struct S3Target { pub bucket_name: String, @@ -53,6 +53,27 @@ impl Display for TraversingDepth { } } +#[derive(ValueEnum, Clone, Copy, Eq, PartialEq, Debug, Serialize, Deserialize)] +pub enum NodeKind { + Safekeeper, + Pageserver, +} + +impl NodeKind { + fn as_str(&self) -> &'static str { + match self { + Self::Safekeeper => "safekeeper", + Self::Pageserver => "pageserver", + } + } +} + +impl Display for NodeKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + impl S3Target { pub fn with_sub_segment(&self, new_segment: &str) -> Self { let mut new_self = self.clone(); @@ -108,6 +129,7 @@ impl RootTarget { } } +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct BucketConfig { pub region: String, pub bucket: String, @@ -143,31 +165,21 @@ impl BucketConfig { } pub struct ConsoleConfig { - pub admin_api_url: Url, + pub token: String, + pub base_url: Url, } impl ConsoleConfig { pub fn from_env() -> anyhow::Result { - let admin_api_url: Url = env::var("CLOUD_ADMIN_API_URL") + let base_url: Url = env::var("CLOUD_ADMIN_API_URL") .context("'CLOUD_ADMIN_API_URL' param retrieval")? .parse() .context("'CLOUD_ADMIN_API_URL' param parsing")?; - Ok(Self { admin_api_url }) - } -} + let token = env::var(CLOUD_ADMIN_API_TOKEN_ENV_VAR) + .context("'CLOUD_ADMIN_API_TOKEN' environment variable fetch")?; -pub fn get_cloud_admin_api_token_or_exit() -> String { - match env::var(CLOUD_ADMIN_API_TOKEN_ENV_VAR) { - Ok(token) => token, - Err(env::VarError::NotPresent) => { - error!("{CLOUD_ADMIN_API_TOKEN_ENV_VAR} env variable is not present"); - std::process::exit(1); - } - Err(env::VarError::NotUnicode(not_unicode_string)) => { - error!("{CLOUD_ADMIN_API_TOKEN_ENV_VAR} env variable's value is not a valid unicode string: {not_unicode_string:?}"); - std::process::exit(1); - } + Ok(Self { base_url, token }) } } @@ -231,6 +243,29 @@ pub fn init_s3_client(account_id: Option, bucket_region: Region) -> Clie Client::from_conf(builder.build()) } +fn init_remote( + bucket_config: BucketConfig, + node_kind: NodeKind, +) -> anyhow::Result<(Arc, RootTarget)> { + let bucket_region = Region::new(bucket_config.region); + let delimiter = "/".to_string(); + let s3_client = Arc::new(init_s3_client(bucket_config.sso_account_id, bucket_region)); + let s3_root = match node_kind { + NodeKind::Pageserver => RootTarget::Pageserver(S3Target { + bucket_name: bucket_config.bucket, + prefix_in_bucket: ["pageserver", "v1", TENANTS_SEGMENT_NAME, ""].join(&delimiter), + delimiter, + }), + NodeKind::Safekeeper => RootTarget::Safekeeper(S3Target { + bucket_name: bucket_config.bucket, + prefix_in_bucket: ["safekeeper", "v1", "wal", ""].join(&delimiter), + delimiter, + }), + }; + + Ok((s3_client, s3_root)) +} + async fn list_objects_with_retries( s3_client: &Client, s3_target: &S3Target, diff --git a/s3_scrubber/src/main.rs b/s3_scrubber/src/main.rs index 3c60723f88..9a0d6c9ae8 100644 --- a/s3_scrubber/src/main.rs +++ b/s3_scrubber/src/main.rs @@ -1,20 +1,8 @@ -use std::collections::HashMap; -use std::fmt::Display; -use std::num::NonZeroUsize; -use std::sync::Arc; - -use anyhow::Context; -use aws_sdk_s3::config::Region; -use s3_scrubber::cloud_admin_api::CloudAdminApiClient; -use s3_scrubber::delete_batch_producer::DeleteBatchProducer; +use s3_scrubber::garbage::{find_garbage, purge_garbage, PurgeMode}; use s3_scrubber::scan_metadata::scan_metadata; -use s3_scrubber::{ - checks, get_cloud_admin_api_token_or_exit, init_logging, init_s3_client, BucketConfig, - ConsoleConfig, RootTarget, S3Deleter, S3Target, TraversingDepth, CLI_NAME, -}; -use tracing::{info, warn}; +use s3_scrubber::{init_logging, BucketConfig, ConsoleConfig, NodeKind, TraversingDepth}; -use clap::{Parser, Subcommand, ValueEnum}; +use clap::{Parser, Subcommand}; #[derive(Parser)] #[command(author, version, about, long_about = None)] @@ -27,212 +15,45 @@ struct Cli { delete: bool, } -#[derive(ValueEnum, Clone, Copy, Eq, PartialEq)] -enum NodeKind { - Safekeeper, - Pageserver, -} - -impl NodeKind { - fn as_str(&self) -> &'static str { - match self { - Self::Safekeeper => "safekeeper", - Self::Pageserver => "pageserver", - } - } -} - -impl Display for NodeKind { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.as_str()) - } -} - -#[derive(Subcommand)] +#[derive(Subcommand, Debug)] enum Command { - Tidy { + FindGarbage { #[arg(short, long)] node_kind: NodeKind, #[arg(short, long, default_value_t=TraversingDepth::Tenant)] depth: TraversingDepth, - #[arg(short, long, default_value_t = false)] - skip_validation: bool, + #[arg(short, long, default_value_t = String::from("garbage.json"))] + output_path: String, + }, + PurgeGarbage { + #[arg(short, long)] + input_path: String, + #[arg(short, long, default_value_t = PurgeMode::DeletedOnly)] + mode: PurgeMode, }, ScanMetadata {}, } -async fn tidy( - cli: &Cli, - bucket_config: BucketConfig, - console_config: ConsoleConfig, - node_kind: NodeKind, - depth: TraversingDepth, - skip_validation: bool, -) -> anyhow::Result<()> { - let dry_run = !cli.delete; - let file_name = if dry_run { - format!( - "{}_{}_{}__dry.log", - CLI_NAME, - node_kind, - chrono::Utc::now().format("%Y_%m_%d__%H_%M_%S") - ) - } else { - format!( - "{}_{}_{}.log", - CLI_NAME, - node_kind, - chrono::Utc::now().format("%Y_%m_%d__%H_%M_%S") - ) - }; - - let _guard = init_logging(&file_name); - - if dry_run { - info!("Dry run, not removing items for real"); - } else { - warn!("Dry run disabled, removing bucket items for real"); - } - - info!("skip_validation={skip_validation}"); - - info!("Starting extra S3 removal in {bucket_config} for node kind '{node_kind}', traversing depth: {depth:?}"); - - info!("Starting extra tenant S3 removal in {bucket_config} for node kind '{node_kind}'"); - let cloud_admin_api_client = Arc::new(CloudAdminApiClient::new( - get_cloud_admin_api_token_or_exit(), - console_config.admin_api_url, - )); - - let bucket_region = Region::new(bucket_config.region); - let delimiter = "/".to_string(); - let s3_client = Arc::new(init_s3_client(bucket_config.sso_account_id, bucket_region)); - let s3_root = match node_kind { - NodeKind::Pageserver => RootTarget::Pageserver(S3Target { - bucket_name: bucket_config.bucket, - prefix_in_bucket: ["pageserver", "v1", "tenants", ""].join(&delimiter), - delimiter, - }), - NodeKind::Safekeeper => RootTarget::Safekeeper(S3Target { - bucket_name: bucket_config.bucket, - prefix_in_bucket: ["safekeeper", "v1", "wal", ""].join(&delimiter), - delimiter, - }), - }; - - let delete_batch_producer = DeleteBatchProducer::start( - Arc::clone(&cloud_admin_api_client), - Arc::clone(&s3_client), - s3_root.clone(), - depth, - ); - - let s3_deleter = S3Deleter::new( - dry_run, - NonZeroUsize::new(15).unwrap(), - Arc::clone(&s3_client), - delete_batch_producer.subscribe(), - s3_root.clone(), - ); - - let (deleter_task_result, batch_producer_task_result) = - tokio::join!(s3_deleter.remove_all(), delete_batch_producer.join()); - - let deletion_stats = deleter_task_result.context("s3 deletion")?; - info!( - "Deleted {} tenants ({} keys) and {} timelines ({} keys) total. Dry run: {}", - deletion_stats.deleted_tenant_keys.len(), - deletion_stats.deleted_tenant_keys.values().sum::(), - deletion_stats.deleted_timeline_keys.len(), - deletion_stats.deleted_timeline_keys.values().sum::(), - dry_run, - ); - info!( - "Total tenant deletion stats: {:?}", - deletion_stats - .deleted_tenant_keys - .into_iter() - .map(|(id, key)| (id.to_string(), key)) - .collect::>() - ); - info!( - "Total timeline deletion stats: {:?}", - deletion_stats - .deleted_timeline_keys - .into_iter() - .map(|(id, key)| (id.to_string(), key)) - .collect::>() - ); - - let batch_producer_stats = batch_producer_task_result.context("delete batch producer join")?; - info!( - "Total bucket tenants listed: {}; for {} active tenants, timelines checked: {}", - batch_producer_stats.tenants_checked(), - batch_producer_stats.active_tenants(), - batch_producer_stats.timelines_checked() - ); - - if node_kind == NodeKind::Pageserver { - info!("node_kind != pageserver, finish without performing validation step"); - return Ok(()); - } - - if skip_validation { - info!("--skip-validation is set, exiting"); - return Ok(()); - } - - info!("validating active tenants and timelines for pageserver S3 data"); - - // TODO kb real stats for validation + better stats for every place: add and print `min`, `max`, `mean` values at least - let validation_stats = checks::validate_pageserver_active_tenant_and_timelines( - s3_client, - s3_root, - cloud_admin_api_client, - batch_producer_stats, - ) - .await - .context("active tenant and timeline validation")?; - info!("Finished active tenant and timeline validation, correct timelines: {}, timeline validation errors: {}", - validation_stats.normal_timelines.len(), validation_stats.timelines_with_errors.len()); - if !validation_stats.timelines_with_errors.is_empty() { - warn!( - "Validation errors: {:#?}", - validation_stats - .timelines_with_errors - .into_iter() - .map(|(id, errors)| (id.to_string(), format!("{errors:?}"))) - .collect::>() - ); - } - - info!("Done"); - Ok(()) -} - #[tokio::main] async fn main() -> anyhow::Result<()> { let cli = Cli::parse(); let bucket_config = BucketConfig::from_env()?; + let command_log_name = match &cli.command { + Command::ScanMetadata { .. } => "scan", + Command::FindGarbage { .. } => "find-garbage", + Command::PurgeGarbage { .. } => "purge-garbage", + }; + let _guard = init_logging(&format!( + "{}_{}_{}_{}.log", + std::env::args().next().unwrap(), + command_log_name, + bucket_config.bucket, + chrono::Utc::now().format("%Y_%m_%d__%H_%M_%S") + )); + match cli.command { - Command::Tidy { - node_kind, - depth, - skip_validation, - } => { - let console_config = ConsoleConfig::from_env()?; - tidy( - &cli, - bucket_config, - console_config, - node_kind, - depth, - skip_validation, - ) - .await - } Command::ScanMetadata {} => match scan_metadata(bucket_config).await { Err(e) => { tracing::error!("Failed: {e}"); @@ -247,5 +68,16 @@ async fn main() -> anyhow::Result<()> { } } }, + Command::FindGarbage { + node_kind, + depth, + output_path, + } => { + let console_config = ConsoleConfig::from_env()?; + find_garbage(bucket_config, console_config, depth, node_kind, output_path).await + } + Command::PurgeGarbage { input_path, mode } => { + purge_garbage(input_path, mode, !cli.delete).await + } } } diff --git a/s3_scrubber/src/metadata_stream.rs b/s3_scrubber/src/metadata_stream.rs index 4e500a96cf..125a370e90 100644 --- a/s3_scrubber/src/metadata_stream.rs +++ b/s3_scrubber/src/metadata_stream.rs @@ -1,9 +1,9 @@ use anyhow::Context; use async_stream::{stream, try_stream}; -use aws_sdk_s3::Client; +use aws_sdk_s3::{types::ObjectIdentifier, Client}; use tokio_stream::Stream; -use crate::{list_objects_with_retries, RootTarget, TenantId}; +use crate::{list_objects_with_retries, RootTarget, S3Target, TenantId}; use utils::id::{TenantTimelineId, TimelineId}; /// Given an S3 bucket, output a stream of TenantIds discovered via ListObjectsv2 @@ -104,3 +104,34 @@ pub async fn stream_tenant_timelines<'a>( } }) } + +pub(crate) fn stream_listing<'a>( + s3_client: &'a Client, + target: &'a S3Target, +) -> impl Stream> + 'a { + try_stream! { + let mut continuation_token = None; + loop { + let fetch_response = + list_objects_with_retries(s3_client, target, continuation_token.clone()).await?; + + if target.delimiter.is_empty() { + for object_id in fetch_response.contents().unwrap_or_default().iter().filter_map(|object| object.key()).map(|i| + ObjectIdentifier::builder().key(i).build() + ) { + yield object_id; + } + } else { + for prefix in fetch_response.common_prefixes().unwrap_or_default() + .iter().filter_map(|p| p.prefix().map(|k| ObjectIdentifier::builder().key(k).build())) { + yield prefix; + } + } + + match fetch_response.next_continuation_token { + Some(new_token) => continuation_token = Some(new_token), + None => break, + } + } + } +} diff --git a/s3_scrubber/src/s3_deletion.rs b/s3_scrubber/src/s3_deletion.rs deleted file mode 100644 index a03cc65c89..0000000000 --- a/s3_scrubber/src/s3_deletion.rs +++ /dev/null @@ -1,434 +0,0 @@ -use std::collections::BTreeMap; -use std::num::NonZeroUsize; -use std::sync::Arc; -use std::time::Duration; - -use anyhow::Context; -use aws_sdk_s3::types::{Delete, ObjectIdentifier}; -use aws_sdk_s3::Client; -use tokio::sync::mpsc::error::TryRecvError; -use tokio::sync::mpsc::UnboundedReceiver; -use tokio::sync::Mutex; -use tokio::task::JoinSet; -use tracing::{debug, error, info, info_span, Instrument}; - -use crate::delete_batch_producer::DeleteBatch; -use crate::{list_objects_with_retries, RootTarget, S3Target, TenantId, MAX_RETRIES}; -use utils::id::TenantTimelineId; - -pub struct S3Deleter { - dry_run: bool, - concurrent_tasks_count: NonZeroUsize, - delete_batch_receiver: Arc>>, - s3_client: Arc, - s3_target: RootTarget, -} - -impl S3Deleter { - pub fn new( - dry_run: bool, - concurrent_tasks_count: NonZeroUsize, - s3_client: Arc, - delete_batch_receiver: Arc>>, - s3_target: RootTarget, - ) -> Self { - Self { - dry_run, - concurrent_tasks_count, - delete_batch_receiver, - s3_client, - s3_target, - } - } - - pub async fn remove_all(self) -> anyhow::Result { - let mut deletion_tasks = JoinSet::new(); - for id in 0..self.concurrent_tasks_count.get() { - let closure_client = Arc::clone(&self.s3_client); - let closure_s3_target = self.s3_target.clone(); - let closure_batch_receiver = Arc::clone(&self.delete_batch_receiver); - let dry_run = self.dry_run; - deletion_tasks.spawn( - async move { - info!("Task started"); - ( - id, - async move { - let mut task_stats = DeletionStats::default(); - loop { - let mut guard = closure_batch_receiver.lock().await; - let receiver_result = guard.try_recv(); - drop(guard); - match receiver_result { - Ok(batch) => { - let stats = delete_batch( - &closure_client, - &closure_s3_target, - batch, - dry_run, - ) - .await - .context("batch deletion")?; - debug!( - "Batch processed, number of objects deleted per tenant in the batch is: {}, per timeline — {}", - stats.deleted_tenant_keys.len(), - stats.deleted_timeline_keys.len(), - ); - task_stats.merge(stats); - } - Err(TryRecvError::Empty) => { - debug!("No tasks yet, waiting"); - tokio::time::sleep(Duration::from_secs(1)).await; - continue; - } - Err(TryRecvError::Disconnected) => { - info!("Task finished: sender dropped"); - return Ok(task_stats); - } - } - } - } - .in_current_span() - .await, - ) - } - .instrument(info_span!("deletion_task", %id)), - ); - } - - let mut total_stats = DeletionStats::default(); - while let Some(task_result) = deletion_tasks.join_next().await { - match task_result { - Ok((id, Ok(task_stats))) => { - info!("Task {id} completed"); - total_stats.merge(task_stats); - } - Ok((id, Err(e))) => { - error!("Task {id} failed: {e:#}"); - return Err(e); - } - Err(join_error) => anyhow::bail!("Failed to join on a task: {join_error:?}"), - } - } - - Ok(total_stats) - } -} - -/// S3 delete_objects allows up to 1000 keys to be passed in a single request. -/// Yet if you pass too many key requests, apparently S3 could return with OK and -/// actually delete nothing, so keep the number lower. -const MAX_ITEMS_TO_DELETE: usize = 200; - -#[derive(Debug, Default)] -pub struct DeletionStats { - pub deleted_tenant_keys: BTreeMap, - pub deleted_timeline_keys: BTreeMap, -} - -impl DeletionStats { - fn merge(&mut self, other: Self) { - self.deleted_tenant_keys.extend(other.deleted_tenant_keys); - self.deleted_timeline_keys - .extend(other.deleted_timeline_keys); - } -} - -async fn delete_batch( - s3_client: &Client, - s3_target: &RootTarget, - batch: DeleteBatch, - dry_run: bool, -) -> anyhow::Result { - let (deleted_tenant_keys, deleted_timeline_keys) = tokio::join!( - delete_tenants_batch(batch.tenants, s3_target, s3_client, dry_run), - delete_timelines_batch(batch.timelines, s3_target, s3_client, dry_run), - ); - - Ok(DeletionStats { - deleted_tenant_keys: deleted_tenant_keys.context("tenant batch deletion")?, - deleted_timeline_keys: deleted_timeline_keys.context("timeline batch deletion")?, - }) -} - -async fn delete_tenants_batch( - batched_tenants: Vec, - s3_target: &RootTarget, - s3_client: &Client, - dry_run: bool, -) -> Result, anyhow::Error> { - info!("Deleting tenants batch of size {}", batched_tenants.len()); - info!("Tenant ids to remove: {batched_tenants:?}"); - let deleted_keys = delete_elements( - &batched_tenants, - s3_target, - s3_client, - dry_run, - |root_target, tenant_to_delete| root_target.tenant_root(&tenant_to_delete), - ) - .await?; - - if !dry_run { - let mut last_err = None; - for _ in 0..MAX_RETRIES { - match ensure_tenant_batch_deleted(s3_client, s3_target, &batched_tenants).await { - Ok(()) => { - last_err = None; - break; - } - Err(e) => { - error!("Failed to ensure the tenant batch is deleted: {e}"); - last_err = Some(e); - } - } - } - - if let Some(e) = last_err { - anyhow::bail!( - "Failed to ensure that tenant batch is deleted {MAX_RETRIES} times: {e:?}" - ); - } - } - - Ok(deleted_keys) -} - -async fn delete_timelines_batch( - batched_timelines: Vec, - s3_target: &RootTarget, - s3_client: &Client, - dry_run: bool, -) -> Result, anyhow::Error> { - info!( - "Deleting timelines batch of size {}", - batched_timelines.len() - ); - info!( - "Timeline ids to remove: {:?}", - batched_timelines - .iter() - .map(|id| id.to_string()) - .collect::>() - ); - let deleted_keys = delete_elements( - &batched_timelines, - s3_target, - s3_client, - dry_run, - |root_target, timeline_to_delete| root_target.timeline_root(&timeline_to_delete), - ) - .await?; - - if !dry_run { - let mut last_err = None; - for _ in 0..MAX_RETRIES { - match ensure_timeline_batch_deleted(s3_client, s3_target, &batched_timelines).await { - Ok(()) => { - last_err = None; - break; - } - Err(e) => { - error!("Failed to ensure the timelines batch is deleted: {e}"); - last_err = Some(e); - } - } - } - - if let Some(e) = last_err { - anyhow::bail!( - "Failed to ensure that timeline batch is deleted {MAX_RETRIES} times: {e:?}" - ); - } - } - Ok(deleted_keys) -} - -async fn delete_elements( - batched_ids: &Vec, - s3_target: &RootTarget, - s3_client: &Client, - dry_run: bool, - target_producer: impl Fn(&RootTarget, I) -> S3Target, -) -> Result, anyhow::Error> -where - I: Ord + PartialOrd + Copy, -{ - let mut deleted_keys = BTreeMap::new(); - let mut object_ids_to_delete = Vec::with_capacity(MAX_ITEMS_TO_DELETE); - for &id_to_delete in batched_ids { - let mut continuation_token = None; - let mut subtargets = vec![target_producer(s3_target, id_to_delete)]; - while let Some(current_target) = subtargets.pop() { - loop { - let fetch_response = list_objects_with_retries( - s3_client, - ¤t_target, - continuation_token.clone(), - ) - .await?; - - for object_id in fetch_response - .contents() - .unwrap_or_default() - .iter() - .filter_map(|object| object.key()) - .map(|key| ObjectIdentifier::builder().key(key).build()) - { - if object_ids_to_delete.len() >= MAX_ITEMS_TO_DELETE { - let object_ids_for_request = std::mem::replace( - &mut object_ids_to_delete, - Vec::with_capacity(MAX_ITEMS_TO_DELETE), - ); - send_delete_request( - s3_client, - s3_target.bucket_name(), - object_ids_for_request, - dry_run, - ) - .await - .context("object ids deletion")?; - } - - object_ids_to_delete.push(object_id); - *deleted_keys.entry(id_to_delete).or_default() += 1; - } - - subtargets.extend( - fetch_response - .common_prefixes() - .unwrap_or_default() - .iter() - .filter_map(|common_prefix| common_prefix.prefix()) - .map(|prefix| { - let mut new_target = current_target.clone(); - new_target.prefix_in_bucket = prefix.to_string(); - new_target - }), - ); - - match fetch_response.next_continuation_token { - Some(new_token) => continuation_token = Some(new_token), - None => break, - } - } - } - } - if !object_ids_to_delete.is_empty() { - info!("Removing last objects of the batch"); - send_delete_request( - s3_client, - s3_target.bucket_name(), - object_ids_to_delete, - dry_run, - ) - .await - .context("Last object ids deletion")?; - } - Ok(deleted_keys) -} - -pub async fn send_delete_request( - s3_client: &Client, - bucket_name: &str, - ids: Vec, - dry_run: bool, -) -> anyhow::Result<()> { - info!("Removing {} object ids from S3", ids.len()); - info!("Object ids to remove: {ids:?}"); - let delete_request = s3_client - .delete_objects() - .bucket(bucket_name) - .delete(Delete::builder().set_objects(Some(ids)).build()); - if dry_run { - info!("Dry run, skipping the actual removal"); - Ok(()) - } else { - let original_request = delete_request.clone(); - - for _ in 0..MAX_RETRIES { - match delete_request - .clone() - .send() - .await - .context("delete request processing") - { - Ok(delete_response) => { - info!("Delete response: {delete_response:?}"); - match delete_response.errors() { - Some(delete_errors) => { - error!("Delete request returned errors: {delete_errors:?}"); - tokio::time::sleep(Duration::from_secs(1)).await; - } - None => { - info!("Successfully removed an object batch from S3"); - return Ok(()); - } - } - } - Err(e) => { - error!("Failed to send a delete request: {e:#}"); - tokio::time::sleep(Duration::from_secs(1)).await; - } - } - } - - error!("Failed to do deletion, request: {original_request:?}"); - anyhow::bail!("Failed to run deletion request {MAX_RETRIES} times"); - } -} - -async fn ensure_tenant_batch_deleted( - s3_client: &Client, - s3_target: &RootTarget, - batch: &[TenantId], -) -> anyhow::Result<()> { - let mut not_deleted_tenants = Vec::with_capacity(batch.len()); - - for &tenant_id in batch { - let fetch_response = - list_objects_with_retries(s3_client, &s3_target.tenant_root(&tenant_id), None).await?; - - if fetch_response.is_truncated() - || fetch_response.contents().is_some() - || fetch_response.common_prefixes().is_some() - { - error!( - "Tenant {tenant_id} should be deleted, but its list response is {fetch_response:?}" - ); - not_deleted_tenants.push(tenant_id); - } - } - - anyhow::ensure!( - not_deleted_tenants.is_empty(), - "Failed to delete all tenants in a batch. Tenants {not_deleted_tenants:?} should be deleted." - ); - Ok(()) -} - -async fn ensure_timeline_batch_deleted( - s3_client: &Client, - s3_target: &RootTarget, - batch: &[TenantTimelineId], -) -> anyhow::Result<()> { - let mut not_deleted_timelines = Vec::with_capacity(batch.len()); - - for &id in batch { - let fetch_response = - list_objects_with_retries(s3_client, &s3_target.timeline_root(&id), None).await?; - - if fetch_response.is_truncated() - || fetch_response.contents().is_some() - || fetch_response.common_prefixes().is_some() - { - error!("Timeline {id} should be deleted, but its list response is {fetch_response:?}"); - not_deleted_timelines.push(id); - } - } - - anyhow::ensure!( - not_deleted_timelines.is_empty(), - "Failed to delete all timelines in a batch" - ); - Ok(()) -} diff --git a/s3_scrubber/src/scan_metadata.rs b/s3_scrubber/src/scan_metadata.rs index f75d7645a8..33acd41a5b 100644 --- a/s3_scrubber/src/scan_metadata.rs +++ b/s3_scrubber/src/scan_metadata.rs @@ -1,17 +1,15 @@ use std::collections::{HashMap, HashSet}; -use std::sync::Arc; use crate::checks::{ branch_cleanup_and_check_errors, list_timeline_blobs, BlobDataParseResult, S3TimelineBlobData, TimelineAnalysis, }; use crate::metadata_stream::{stream_tenant_timelines, stream_tenants}; -use crate::{init_logging, init_s3_client, BucketConfig, RootTarget, S3Target, CLI_NAME}; +use crate::{init_remote, BucketConfig, NodeKind, RootTarget}; use aws_sdk_s3::Client; -use aws_types::region::Region; use futures_util::{pin_mut, StreamExt, TryStreamExt}; use histogram::Histogram; -use pageserver::tenant::{IndexPart, TENANTS_SEGMENT_NAME}; +use pageserver::tenant::IndexPart; use utils::id::TenantTimelineId; pub struct MetadataSummary { @@ -175,25 +173,7 @@ Timeline layer count: {6} /// Scan the pageserver metadata in an S3 bucket, reporting errors and statistics. pub async fn scan_metadata(bucket_config: BucketConfig) -> anyhow::Result { - let file_name = format!( - "{}_scan_metadata_{}_{}.log", - CLI_NAME, - bucket_config.bucket, - chrono::Utc::now().format("%Y_%m_%d__%H_%M_%S") - ); - - let _guard = init_logging(&file_name); - - let s3_client = Arc::new(init_s3_client( - bucket_config.sso_account_id, - Region::new(bucket_config.region), - )); - let delimiter = "/"; - let target = RootTarget::Pageserver(S3Target { - bucket_name: bucket_config.bucket.to_string(), - prefix_in_bucket: ["pageserver", "v1", TENANTS_SEGMENT_NAME, ""].join(delimiter), - delimiter: delimiter.to_string(), - }); + let (s3_client, target) = init_remote(bucket_config, NodeKind::Pageserver)?; let tenants = stream_tenants(&s3_client, &target); From 71611f4ab3ae7d638ac03a037dee4860a5c9d5c6 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Thu, 26 Oct 2023 22:54:37 +0100 Subject: [PATCH 32/69] proxy: prepare to remove high cardinality metrics (#5461) ## Problem High cardinality metrics are bad ## Summary of changes Preparing to remove high cardinality metrics. Will actually remove in #5466 --- proxy/src/proxy.rs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index 165794414f..9a01a27fc4 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -171,7 +171,7 @@ static NUM_WAKEUP_FAILURES: Lazy = Lazy::new(|| { .unwrap() }); -static NUM_BYTES_PROXIED_COUNTER: Lazy = Lazy::new(|| { +static NUM_BYTES_PROXIED_PER_CLIENT_COUNTER: Lazy = Lazy::new(|| { register_int_counter_vec!( "proxy_io_bytes_per_client", "Number of bytes sent/received between client and backend.", @@ -180,6 +180,15 @@ static NUM_BYTES_PROXIED_COUNTER: Lazy = Lazy::new(|| { .unwrap() }); +static NUM_BYTES_PROXIED_COUNTER: Lazy = Lazy::new(|| { + register_int_counter_vec!( + "proxy_io_bytes", + "Number of bytes sent/received between all clients and backends.", + &["direction"], + ) + .unwrap() +}); + pub async fn task_main( config: &'static ProxyConfig, listener: tokio::net::TcpListener, @@ -764,24 +773,28 @@ pub async fn proxy_pass( branch_id: aux.branch_id.to_string(), }); - let m_sent = NUM_BYTES_PROXIED_COUNTER.with_label_values(&aux.traffic_labels("tx")); + let m_sent = NUM_BYTES_PROXIED_COUNTER.with_label_values(&["tx"]); + let m_sent2 = NUM_BYTES_PROXIED_PER_CLIENT_COUNTER.with_label_values(&aux.traffic_labels("tx")); let mut client = MeasuredStream::new( client, |_| {}, |cnt| { // Number of bytes we sent to the client (outbound). m_sent.inc_by(cnt as u64); + m_sent2.inc_by(cnt as u64); usage.record_egress(cnt as u64); }, ); - let m_recv = NUM_BYTES_PROXIED_COUNTER.with_label_values(&aux.traffic_labels("rx")); + let m_recv = NUM_BYTES_PROXIED_COUNTER.with_label_values(&["rx"]); + let m_recv2 = NUM_BYTES_PROXIED_PER_CLIENT_COUNTER.with_label_values(&aux.traffic_labels("rx")); let mut compute = MeasuredStream::new( compute, |_| {}, |cnt| { // Number of bytes the client sent to the compute node (inbound). m_recv.inc_by(cnt as u64); + m_recv2.inc_by(cnt as u64); }, ); From 83567f9e4e95f79c5298a30c4bc57125c87548fd Mon Sep 17 00:00:00 2001 From: John Spray Date: Fri, 27 Oct 2023 09:23:49 +0100 Subject: [PATCH 33/69] tests: revise perf test that interfered with local disk state (#5682) This benchmark started failing after #5580 merged. It was manually deleting some local content on a pageserver, and expecting the behavior that the pageserver would "forget" about the timeline on startup as a result. That is no longer our behavior: pageservers use the remote storage as the source of truth. Rather than having the test go manually delete things at all, we can just delete the whole tenant via the pageserver API, and thereby start from a clean situation. --- test_runner/performance/test_bulk_insert.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/test_runner/performance/test_bulk_insert.py b/test_runner/performance/test_bulk_insert.py index aafc315576..62301f3919 100644 --- a/test_runner/performance/test_bulk_insert.py +++ b/test_runner/performance/test_bulk_insert.py @@ -1,7 +1,7 @@ -import shutil from contextlib import closing from fixtures.compare_fixtures import NeonCompare, PgCompare +from fixtures.pageserver.utils import wait_tenant_status_404 from fixtures.pg_version import PgVersion @@ -42,17 +42,14 @@ def measure_recovery_time(env: NeonCompare): client = env.env.pageserver.http_client() pg_version = PgVersion(client.timeline_detail(env.tenant, env.timeline)["pg_version"]) - # Stop pageserver and remove tenant data - env.env.pageserver.stop() - timeline_dir = env.env.pageserver.timeline_dir(env.tenant, env.timeline) - shutil.rmtree(timeline_dir) - - # Start pageserver - env.env.pageserver.start() + # Delete the Tenant in the pageserver: this will drop local and remote layers, such that + # when we "create" the Tenant again, we will replay the WAL from the beginning. + client.tenant_delete(env.tenant) + wait_tenant_status_404(client, env.tenant, iterations=60, interval=0.5) + client.tenant_create(new_tenant_id=env.tenant) # Measure recovery time with env.record_duration("wal_recovery"): - # Create the tenant, which will start walingest client.timeline_create(pg_version, env.tenant, env.timeline) # Flush, which will also wait for lsn to catch up From 39f8fd69456ba59e11fec9e6a52beb58f011bef3 Mon Sep 17 00:00:00 2001 From: duguorong009 <80258679+duguorong009@users.noreply.github.com> Date: Fri, 27 Oct 2023 05:47:11 -0400 Subject: [PATCH 34/69] feat: add `build_tag` env support for `set_build_info_metric` (#5576) - Add a new util `project_build_tag` macro, similar to `project_git_version` - Update the `set_build_info_metric` to accept and make use of `build_tag` info - Update all codes which use the `set_build_info_metric` --- libs/metrics/src/lib.rs | 6 +++--- libs/utils/src/lib.rs | 15 +++++++++++++++ pageserver/src/bin/pageserver.rs | 12 +++++++----- proxy/src/bin/proxy.rs | 6 ++++-- safekeeper/src/bin/safekeeper.rs | 6 ++++-- storage_broker/src/bin/storage_broker.rs | 6 ++++-- test_runner/regress/test_build_info_metric.py | 3 +++ 7 files changed, 40 insertions(+), 14 deletions(-) diff --git a/libs/metrics/src/lib.rs b/libs/metrics/src/lib.rs index b4026d93e1..0b3b24c18e 100644 --- a/libs/metrics/src/lib.rs +++ b/libs/metrics/src/lib.rs @@ -89,14 +89,14 @@ pub const DISK_WRITE_SECONDS_BUCKETS: &[f64] = &[ 0.000_050, 0.000_100, 0.000_500, 0.001, 0.003, 0.005, 0.01, 0.05, 0.1, 0.3, 0.5, ]; -pub fn set_build_info_metric(revision: &str) { +pub fn set_build_info_metric(revision: &str, build_tag: &str) { let metric = register_int_gauge_vec!( "libmetrics_build_info", "Build/version information", - &["revision"] + &["revision", "build_tag"] ) .expect("Failed to register build info metric"); - metric.with_label_values(&[revision]).set(1); + metric.with_label_values(&[revision, build_tag]).set(1); } // Records I/O stats in a "cross-platform" way. diff --git a/libs/utils/src/lib.rs b/libs/utils/src/lib.rs index b3e38cd9e1..88cefd516d 100644 --- a/libs/utils/src/lib.rs +++ b/libs/utils/src/lib.rs @@ -130,6 +130,21 @@ macro_rules! project_git_version { }; } +/// This is a shortcut to embed build tag into binaries and avoid copying the same build script to all packages +#[macro_export] +macro_rules! project_build_tag { + ($const_identifier:ident) => { + const $const_identifier: &::core::primitive::str = { + const __ARG: &[&::core::primitive::str; 2] = &match ::core::option_env!("BUILD_TAG") { + ::core::option::Option::Some(x) => ["build_tag-env:", x], + ::core::option::Option::None => ["build_tag:", ""], + }; + + $crate::__const_format::concatcp!(__ARG[0], __ARG[1]) + }; + }; +} + /// Re-export for `project_git_version` macro #[doc(hidden)] pub use const_format as __const_format; diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 798b9f258b..df2db25ec6 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -34,11 +34,12 @@ use postgres_backend::AuthType; use utils::logging::TracingErrorLayerEnablement; use utils::signals::ShutdownSignals; use utils::{ - auth::JwtAuth, logging, project_git_version, sentry_init::init_sentry, signals::Signal, - tcp_listener, + auth::JwtAuth, logging, project_build_tag, project_git_version, sentry_init::init_sentry, + signals::Signal, tcp_listener, }; project_git_version!(GIT_VERSION); +project_build_tag!(BUILD_TAG); const PID_FILE_NAME: &str = "pageserver.pid"; @@ -258,11 +259,12 @@ fn start_pageserver( // A changed version string indicates changed software. // A changed launch timestamp indicates a pageserver restart. info!( - "version: {} launch_timestamp: {}", + "version: {} launch_timestamp: {} build_tag: {}", version(), - launch_ts.to_string() + launch_ts.to_string(), + BUILD_TAG, ); - set_build_info_metric(GIT_VERSION); + set_build_info_metric(GIT_VERSION, BUILD_TAG); set_launch_timestamp_metric(launch_ts); pageserver::preinitialize_metrics(); diff --git a/proxy/src/bin/proxy.rs b/proxy/src/bin/proxy.rs index a9ca308797..28d0d95c5b 100644 --- a/proxy/src/bin/proxy.rs +++ b/proxy/src/bin/proxy.rs @@ -16,9 +16,10 @@ use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; use tracing::info; use tracing::warn; -use utils::{project_git_version, sentry_init::init_sentry}; +use utils::{project_build_tag, project_git_version, sentry_init::init_sentry}; project_git_version!(GIT_VERSION); +project_build_tag!(BUILD_TAG); use clap::{Parser, ValueEnum}; @@ -100,7 +101,8 @@ async fn main() -> anyhow::Result<()> { let _sentry_guard = init_sentry(Some(GIT_VERSION.into()), &[]); info!("Version: {GIT_VERSION}"); - ::metrics::set_build_info_metric(GIT_VERSION); + info!("Build_tag: {BUILD_TAG}"); + ::metrics::set_build_info_metric(GIT_VERSION, BUILD_TAG); let args = ProxyCliArgs::parse(); let config = build_config(&args)?; diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index 202343fca1..d476cf33ae 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -42,7 +42,7 @@ use utils::auth::{JwtAuth, Scope}; use utils::{ id::NodeId, logging::{self, LogFormat}, - project_git_version, + project_build_tag, project_git_version, sentry_init::init_sentry, tcp_listener, }; @@ -51,6 +51,7 @@ const PID_FILE_NAME: &str = "safekeeper.pid"; const ID_FILE_NAME: &str = "safekeeper.id"; project_git_version!(GIT_VERSION); +project_build_tag!(BUILD_TAG); const ABOUT: &str = r#" A fleet of safekeepers is responsible for reliably storing WAL received from @@ -204,6 +205,7 @@ async fn main() -> anyhow::Result<()> { )?; logging::replace_panic_hook_with_tracing_panic_hook().forget(); info!("version: {GIT_VERSION}"); + info!("buld_tag: {BUILD_TAG}"); let args_workdir = &args.datadir; let workdir = args_workdir.canonicalize_utf8().with_context(|| { @@ -423,7 +425,7 @@ async fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { .map(|res| ("WAL remover".to_owned(), res)); tasks_handles.push(Box::pin(wal_remover_handle)); - set_build_info_metric(GIT_VERSION); + set_build_info_metric(GIT_VERSION, BUILD_TAG); // TODO: update tokio-stream, convert to real async Stream with // SignalStream, map it to obtain missing signal name, combine streams into diff --git a/storage_broker/src/bin/storage_broker.rs b/storage_broker/src/bin/storage_broker.rs index 23d1bee354..5ce24a6a42 100644 --- a/storage_broker/src/bin/storage_broker.rs +++ b/storage_broker/src/bin/storage_broker.rs @@ -44,10 +44,11 @@ use storage_broker::{ }; use utils::id::TenantTimelineId; use utils::logging::{self, LogFormat}; -use utils::project_git_version; use utils::sentry_init::init_sentry; +use utils::{project_build_tag, project_git_version}; project_git_version!(GIT_VERSION); +project_build_tag!(BUILD_TAG); const DEFAULT_CHAN_SIZE: usize = 32; const DEFAULT_ALL_KEYS_CHAN_SIZE: usize = 16384; @@ -438,7 +439,8 @@ async fn main() -> Result<(), Box> { // initialize sentry if SENTRY_DSN is provided let _sentry_guard = init_sentry(Some(GIT_VERSION.into()), &[]); info!("version: {GIT_VERSION}"); - ::metrics::set_build_info_metric(GIT_VERSION); + info!("build_tag: {BUILD_TAG}"); + metrics::set_build_info_metric(GIT_VERSION, BUILD_TAG); // On any shutdown signal, log receival and exit. std::thread::spawn(move || { diff --git a/test_runner/regress/test_build_info_metric.py b/test_runner/regress/test_build_info_metric.py index 4e53928d14..8f714dae67 100644 --- a/test_runner/regress/test_build_info_metric.py +++ b/test_runner/regress/test_build_info_metric.py @@ -17,3 +17,6 @@ def test_build_info_metric(neon_env_builder: NeonEnvBuilder, link_proxy: NeonPro assert "revision" in sample.labels assert len(sample.labels["revision"]) > 0 + + assert "build_tag" in sample.labels + assert len(sample.labels["build_tag"]) > 0 From 68f15cf967d36936341477c9ccdae38ebab68e8d Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Fri, 27 Oct 2023 13:16:01 +0300 Subject: [PATCH 35/69] fix: schedule_compaction_update must only unlink (#5675) #5649 added the concept of dangling layers which #4938 uses but only partially. I forgot to change `schedule_compaction_update` to not schedule deletions to uphold the "have a layer, you can read it". With the now remembered fix, I don't think these checks should ever fail except for a mistake I already did. These changes might be useful for protecting future changes, even though the Layer carrying the generation AND the `schedule_(gc|compaction)_update` require strong arcs. Rationale for keeping the `#[cfg(feature = "testing")]` is worsening any leak situation which might come up. --- .../src/tenant/remote_timeline_client.rs | 30 +++++++++++++++++-- pageserver/src/tenant/upload_queue.rs | 12 ++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index d91970a1c0..76e75253fb 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -721,6 +721,17 @@ impl RemoteTimelineClient { }) .collect(); + #[cfg(feature = "testing")] + for (name, gen) in &with_generations { + if let Some(unexpected) = upload_queue.dangling_files.insert(name.to_owned(), *gen) { + if &unexpected == gen { + tracing::error!("{name} was unlinked twice with same generation"); + } else { + tracing::error!("{name} was unlinked twice with different generations {gen:?} and {unexpected:?}"); + } + } + } + // after unlinking files from the upload_queue.latest_files we must always schedule an // index_part update, because that needs to be uploaded before we can actually delete the // files. @@ -754,6 +765,19 @@ impl RemoteTimelineClient { info!("scheduling deletion of layer {}{}", name, gen.get_suffix()); } + #[cfg(feature = "testing")] + for (name, gen) in &with_generations { + match upload_queue.dangling_files.remove(name) { + Some(same) if &same == gen => { /* expected */ } + Some(other) => { + tracing::error!("{name} was unlinked with {other:?} but deleted with {gen:?}"); + } + None => { + tracing::error!("{name} was unlinked but was not dangling"); + } + } + } + // schedule the actual deletions let op = UploadOp::Delete(Delete { layers: with_generations, @@ -779,9 +803,7 @@ impl RemoteTimelineClient { let names = compacted_from.iter().map(|x| x.layer_desc().filename()); - let with_generations = - self.schedule_unlinking_of_layers_from_index_part0(upload_queue, names); - self.schedule_deletion_of_unlinked0(upload_queue, with_generations); + self.schedule_unlinking_of_layers_from_index_part0(upload_queue, names); self.launch_queued_tasks(upload_queue); Ok(()) @@ -1434,6 +1456,8 @@ impl RemoteTimelineClient { num_inprogress_deletions: 0, inprogress_tasks: HashMap::default(), queued_operations: VecDeque::default(), + #[cfg(feature = "testing")] + dangling_files: HashMap::default(), }; let upload_queue = std::mem::replace( diff --git a/pageserver/src/tenant/upload_queue.rs b/pageserver/src/tenant/upload_queue.rs index 5959819548..b47878aacc 100644 --- a/pageserver/src/tenant/upload_queue.rs +++ b/pageserver/src/tenant/upload_queue.rs @@ -80,6 +80,14 @@ pub(crate) struct UploadQueueInitialized { /// tasks to finish. For example, metadata upload cannot be performed before all /// preceding layer file uploads have completed. pub(crate) queued_operations: VecDeque, + + /// Files which have been unlinked but not yet had scheduled a deletion for. Only kept around + /// for error logging. + /// + /// Putting this behind a testing feature to catch problems in tests, but assuming we could have a + /// bug causing leaks, then it's better to not leave this enabled for production builds. + #[cfg(feature = "testing")] + pub(crate) dangling_files: HashMap, } impl UploadQueueInitialized { @@ -136,6 +144,8 @@ impl UploadQueue { num_inprogress_deletions: 0, inprogress_tasks: HashMap::new(), queued_operations: VecDeque::new(), + #[cfg(feature = "testing")] + dangling_files: HashMap::new(), }; *self = UploadQueue::Initialized(state); @@ -181,6 +191,8 @@ impl UploadQueue { num_inprogress_deletions: 0, inprogress_tasks: HashMap::new(), queued_operations: VecDeque::new(), + #[cfg(feature = "testing")] + dangling_files: HashMap::new(), }; *self = UploadQueue::Initialized(state); From 262348e41b04dab7643c2854f4af0cd4a8a20a58 Mon Sep 17 00:00:00 2001 From: Arthur Petukhovsky Date: Fri, 27 Oct 2023 12:09:02 +0100 Subject: [PATCH 36/69] Fix safekeeper log spans (#5643) We were missing spans with ttid in "WAL backup" and several other places, this commit should fix it. Here are the examples of logs before and after: https://gist.github.com/petuhovskiy/711a4a4e7ddde3cab3fa6419b2f70fb9 --- safekeeper/src/handler.rs | 16 +++++++----- safekeeper/src/remove_wal.rs | 22 +++++++--------- safekeeper/src/wal_backup.rs | 48 +++++++++++++++++++---------------- safekeeper/src/wal_service.rs | 14 +++++----- 4 files changed, 51 insertions(+), 49 deletions(-) diff --git a/safekeeper/src/handler.rs b/safekeeper/src/handler.rs index 71b99ab1d8..6d0ed8650d 100644 --- a/safekeeper/src/handler.rs +++ b/safekeeper/src/handler.rs @@ -140,6 +140,12 @@ impl postgres_backend::Handler } } + let ttid = TenantTimelineId::new( + self.tenant_id.unwrap_or(TenantId::from([0u8; 16])), + self.timeline_id.unwrap_or(TimelineId::from([0u8; 16])), + ); + tracing::Span::current().record("ttid", tracing::field::display(ttid)); + Ok(()) } else { Err(QueryError::Other(anyhow::anyhow!( @@ -208,26 +214,22 @@ impl postgres_backend::Handler PG_QUERIES_FINISHED.with_label_values(&[cmd_str]).inc(); } - info!( - "got query {:?} in timeline {:?}", - query_string, self.timeline_id - ); + info!("got query {:?}", query_string); let tenant_id = self.tenant_id.context("tenantid is required")?; let timeline_id = self.timeline_id.context("timelineid is required")?; self.check_permission(Some(tenant_id))?; self.ttid = TenantTimelineId::new(tenant_id, timeline_id); - let span_ttid = self.ttid; // satisfy borrow checker match cmd { SafekeeperPostgresCommand::StartWalPush => { self.handle_start_wal_push(pgb) - .instrument(info_span!("WAL receiver", ttid = %span_ttid)) + .instrument(info_span!("WAL receiver")) .await } SafekeeperPostgresCommand::StartReplication { start_lsn, term } => { self.handle_start_replication(pgb, start_lsn, term) - .instrument(info_span!("WAL sender", ttid = %span_ttid)) + .instrument(info_span!("WAL sender")) .await } SafekeeperPostgresCommand::IdentifySystem => self.handle_identify_system(pgb).await, diff --git a/safekeeper/src/remove_wal.rs b/safekeeper/src/remove_wal.rs index 3306f0b63a..d96eedf401 100644 --- a/safekeeper/src/remove_wal.rs +++ b/safekeeper/src/remove_wal.rs @@ -16,20 +16,16 @@ pub async fn task_main(conf: SafeKeeperConf) -> anyhow::Result<()> { continue; } let ttid = tli.ttid; - if let Err(e) = tli - .maybe_persist_control_file() - .instrument(info_span!("", tenant = %ttid.tenant_id, timeline = %ttid.timeline_id)) - .await - { - warn!("failed to persist control file: {e}"); - } - if let Err(e) = tli - .remove_old_wal(conf.wal_backup_enabled) - .instrument(info_span!("", tenant = %ttid.tenant_id, timeline = %ttid.timeline_id)) - .await - { - error!("failed to remove WAL: {}", e); + async { + if let Err(e) = tli.maybe_persist_control_file().await { + warn!("failed to persist control file: {e}"); + } + if let Err(e) = tli.remove_old_wal(conf.wal_backup_enabled).await { + error!("failed to remove WAL: {}", e); + } } + .instrument(info_span!("WAL removal", ttid = %ttid)) + .await; } sleep(wal_removal_interval).await; } diff --git a/safekeeper/src/wal_backup.rs b/safekeeper/src/wal_backup.rs index da8c197411..22c68ce3c9 100644 --- a/safekeeper/src/wal_backup.rs +++ b/safekeeper/src/wal_backup.rs @@ -136,7 +136,7 @@ async fn update_task( if elected_me != (entry.handle.is_some()) { if elected_me { - info!("elected for backup {}: {}", ttid, election_dbg_str); + info!("elected for backup: {}", election_dbg_str); let (shutdown_tx, shutdown_rx) = mpsc::channel(1); let timeline_dir = conf.timeline_dir(&ttid); @@ -149,7 +149,7 @@ async fn update_task( conf.backup_parallel_jobs, shutdown_rx, ) - .instrument(info_span!("WAL backup task", ttid = %ttid)), + .in_current_span(), ); entry.handle = Some(WalBackupTaskHandle { @@ -157,7 +157,7 @@ async fn update_task( handle, }); } else { - info!("stepping down from backup {}: {}", ttid, election_dbg_str); + info!("stepping down from backup: {}", election_dbg_str); shut_down_task(ttid, entry).await; } } @@ -199,29 +199,33 @@ pub async fn wal_backup_launcher_task_main( if conf.remote_storage.is_none() || !conf.wal_backup_enabled { continue; /* just drain the channel and do nothing */ } - let timeline = is_wal_backup_required(ttid).await; - // do we need to do anything at all? - if timeline.is_some() != tasks.contains_key(&ttid) { - if let Some(timeline) = timeline { - // need to start the task - let entry = tasks.entry(ttid).or_insert(WalBackupTimelineEntry { - timeline, - handle: None, - }); - update_task(&conf, ttid, entry).await; - } else { - // need to stop the task - info!("stopping WAL backup task for {}", ttid); - let mut entry = tasks.remove(&ttid).unwrap(); - shut_down_task(ttid, &mut entry).await; + async { + let timeline = is_wal_backup_required(ttid).await; + // do we need to do anything at all? + if timeline.is_some() != tasks.contains_key(&ttid) { + if let Some(timeline) = timeline { + // need to start the task + let entry = tasks.entry(ttid).or_insert(WalBackupTimelineEntry { + timeline, + handle: None, + }); + update_task(&conf, ttid, entry).await; + } else { + // need to stop the task + info!("stopping WAL backup task"); + let mut entry = tasks.remove(&ttid).unwrap(); + shut_down_task(ttid, &mut entry).await; + } } - } + }.instrument(info_span!("WAL backup", ttid = %ttid)).await; } // For each timeline needing offloading, check if this safekeeper // should do the job and start/stop the task accordingly. _ = ticker.tick() => { for (ttid, entry) in tasks.iter_mut() { - update_task(&conf, *ttid, entry).await; + update_task(&conf, *ttid, entry) + .instrument(info_span!("WAL backup", ttid = %ttid)) + .await; } } } @@ -248,7 +252,7 @@ async fn backup_task_main( info!("started"); let res = GlobalTimelines::get(ttid); if let Err(e) = res { - error!("backup error for timeline {}: {}", ttid, e); + error!("backup error: {}", e); return; } let tli = res.unwrap(); @@ -346,7 +350,7 @@ impl WalBackupTask { } } -pub async fn backup_lsn_range( +async fn backup_lsn_range( timeline: &Arc, backup_lsn: &mut Lsn, end_lsn: Lsn, diff --git a/safekeeper/src/wal_service.rs b/safekeeper/src/wal_service.rs index 9fabaa79fb..bceaad1e16 100644 --- a/safekeeper/src/wal_service.rs +++ b/safekeeper/src/wal_service.rs @@ -36,14 +36,14 @@ pub async fn task_main( let conf = conf.clone(); let conn_id = issue_connection_id(&mut connection_count); - tokio::spawn(async move { - if let Err(err) = handle_socket(socket, conf, conn_id, allowed_auth_scope) - .instrument(info_span!("", cid = %conn_id)) - .await - { - error!("connection handler exited: {}", err); + tokio::spawn( + async move { + if let Err(err) = handle_socket(socket, conf, conn_id, allowed_auth_scope).await { + error!("connection handler exited: {}", err); + } } - }); + .instrument(info_span!("", cid = %conn_id, ttid = field::Empty)), + ); } } From a5292f7e67727307341c1d4a372ee7f69efbb626 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Fri, 27 Oct 2023 14:36:34 +0300 Subject: [PATCH 37/69] Some minor renames in attachment service API (#5687) ## Problem ## Summary of changes ## Checklist before requesting a review - [x] I have performed a self-review of my code. - [ ] ~~If it is a core feature, I have added thorough tests.~~ - [ ] ~~Do we need to implement analytics? if so did you add the relevant metrics to the dashboard?~~ - [ ] ~~If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section.~~ ## Checklist before merging - [ ] Do not forget to reformat commit message to not include the above checklist --- control_plane/src/attachment_service.rs | 6 +++--- control_plane/src/bin/attachment_service.rs | 12 ++++++------ libs/pageserver_api/src/control_api.rs | 2 +- pageserver/src/control_plane_client.rs | 2 +- test_runner/fixtures/neon_fixtures.py | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/control_plane/src/attachment_service.rs b/control_plane/src/attachment_service.rs index d7828cdba7..ca632c5eb6 100644 --- a/control_plane/src/attachment_service.rs +++ b/control_plane/src/attachment_service.rs @@ -19,7 +19,7 @@ const COMMAND: &str = "attachment_service"; pub struct AttachHookRequest { #[serde_as(as = "DisplayFromStr")] pub tenant_id: TenantId, - pub pageserver_id: Option, + pub node_id: Option, } #[derive(Serialize, Deserialize)] @@ -85,7 +85,7 @@ impl AttachmentService { .control_plane_api .clone() .unwrap() - .join("attach_hook") + .join("attach-hook") .unwrap(); let client = reqwest::blocking::ClientBuilder::new() .build() @@ -93,7 +93,7 @@ impl AttachmentService { let request = AttachHookRequest { tenant_id, - pageserver_id: Some(pageserver_id), + node_id: Some(pageserver_id), }; let response = client.post(url).json(&request).send()?; diff --git a/control_plane/src/bin/attachment_service.rs b/control_plane/src/bin/attachment_service.rs index 65ed78d20b..0ccd9e75fc 100644 --- a/control_plane/src/bin/attachment_service.rs +++ b/control_plane/src/bin/attachment_service.rs @@ -171,7 +171,7 @@ async fn handle_re_attach(mut req: Request) -> Result, ApiE state.generation += 1; response.tenants.push(ReAttachResponseTenant { id: *t, - generation: state.generation, + gen: state.generation, }); } } @@ -217,14 +217,14 @@ async fn handle_attach_hook(mut req: Request) -> Result, Ap .tenants .entry(attach_req.tenant_id) .or_insert_with(|| TenantState { - pageserver: attach_req.pageserver_id, + pageserver: attach_req.node_id, generation: 0, }); - if attach_req.pageserver_id.is_some() { + if attach_req.node_id.is_some() { tenant_state.generation += 1; } - tenant_state.pageserver = attach_req.pageserver_id; + tenant_state.pageserver = attach_req.node_id; let generation = tenant_state.generation; locked.save().await.map_err(ApiError::InternalServerError)?; @@ -232,7 +232,7 @@ async fn handle_attach_hook(mut req: Request) -> Result, Ap json_response( StatusCode::OK, AttachHookResponse { - gen: attach_req.pageserver_id.map(|_| generation), + gen: attach_req.node_id.map(|_| generation), }, ) } @@ -242,7 +242,7 @@ fn make_router(persistent_state: PersistentState) -> RouterBuilder>()) } diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 7a3e0a9efc..81a7b0750d 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1554,8 +1554,8 @@ class NeonAttachmentService: def attach_hook(self, tenant_id: TenantId, pageserver_id: int) -> int: response = requests.post( - f"{self.env.control_plane_api}/attach_hook", - json={"tenant_id": str(tenant_id), "pageserver_id": pageserver_id}, + f"{self.env.control_plane_api}/attach-hook", + json={"tenant_id": str(tenant_id), "node_id": pageserver_id}, ) response.raise_for_status() gen = response.json()["gen"] From c13e932c3bd8096669619326ab8decdfd2ffca20 Mon Sep 17 00:00:00 2001 From: John Spray Date: Fri, 27 Oct 2023 14:20:04 +0100 Subject: [PATCH 38/69] pageserver: add generation fields in openapi spec (#5690) These optional fields have existed for as while, but weren't mentioned in `openapi_spec.yaml` yet. --- pageserver/src/http/openapi_spec.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index e8421d619d..2d954f6823 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -713,6 +713,12 @@ paths: Errors if the tenant is absent on disk, already present in memory or fails to schedule its load. Scheduling a load does not mean that the tenant would load successfully, check tenant status to ensure load correctness. + requestBody: + required: false + content: + application/json: + schema: + $ref: "#/components/schemas/TenantLoadRequest" responses: "202": description: Tenant scheduled to load successfully @@ -1203,6 +1209,15 @@ components: new_tenant_id: type: string format: hex + generation: + type: integer + description: Attachment generation number. + TenantLoadRequest: + type: object + properties: + generation: + type: integer + description: Attachment generation number. TenantAttachRequest: type: object required: From 493b47e1da97b028423396c37ee3ec936b714323 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Fri, 27 Oct 2023 18:17:39 +0100 Subject: [PATCH 39/69] proxy: exclude client latencies in metrics (#5688) ## Problem In #5539, I moved the connect_to_compute latency to start counting before authentication - this is because authentication will perform some calls to the control plane in order to get credentials and to eagerly wake a compute server. It felt important to include these times in the latency metric as these are times we should definitely care about reducing. What is not interesting to record in this metric is the roundtrip time during authentication when we wait for the client to respond. ## Summary of changes Implement a mechanism to pause the latency timer, resuming on drop of the pause struct. We pause the timer right before we send the authentication message to the client, and we resume the timer right after we complete the authentication flow. --- proxy/src/auth/backend.rs | 44 ++++++++++++++++++++++++++----- proxy/src/auth/backend/classic.rs | 26 ++++++++++-------- proxy/src/auth/backend/hacks.rs | 11 ++++++++ proxy/src/proxy.rs | 42 ++++++++++++++++++++++++----- 4 files changed, 99 insertions(+), 24 deletions(-) diff --git a/proxy/src/auth/backend.rs b/proxy/src/auth/backend.rs index 850d3a1837..9cf45c0eec 100644 --- a/proxy/src/auth/backend.rs +++ b/proxy/src/auth/backend.rs @@ -5,7 +5,7 @@ mod link; pub use link::LinkAuthError; use tokio_postgres::config::AuthKeys; -use crate::proxy::{handle_try_wake, retry_after}; +use crate::proxy::{handle_try_wake, retry_after, LatencyTimer}; use crate::{ auth::{self, ClientCredentials}, config::AuthenticationConfig, @@ -134,13 +134,14 @@ async fn auth_quirks_creds( client: &mut stream::PqStream, allow_cleartext: bool, config: &'static AuthenticationConfig, + latency_timer: &mut LatencyTimer, ) -> auth::Result> { // If there's no project so far, that entails that client doesn't // support SNI or other means of passing the endpoint (project) name. // We now expect to see a very specific payload in the place of password. if creds.project.is_none() { // Password will be checked by the compute node later. - return hacks::password_hack(creds, client).await; + return hacks::password_hack(creds, client, latency_timer).await; } // Password hack should set the project name. @@ -151,11 +152,11 @@ async fn auth_quirks_creds( // Currently, we use it for websocket connections (latency). if allow_cleartext { // Password will be checked by the compute node later. - return hacks::cleartext_hack(client).await; + return hacks::cleartext_hack(client, latency_timer).await; } // Finally, proceed with the main auth flow (SCRAM-based). - classic::authenticate(api, extra, creds, client, config).await + classic::authenticate(api, extra, creds, client, config, latency_timer).await } /// True to its name, this function encapsulates our current auth trade-offs. @@ -167,8 +168,18 @@ async fn auth_quirks( client: &mut stream::PqStream, allow_cleartext: bool, config: &'static AuthenticationConfig, + latency_timer: &mut LatencyTimer, ) -> auth::Result> { - let auth_stuff = auth_quirks_creds(api, extra, creds, client, allow_cleartext, config).await?; + let auth_stuff = auth_quirks_creds( + api, + extra, + creds, + client, + allow_cleartext, + config, + latency_timer, + ) + .await?; let mut num_retries = 0; let mut node = loop { @@ -233,6 +244,7 @@ impl BackendType<'_, ClientCredentials<'_>> { client: &mut stream::PqStream, allow_cleartext: bool, config: &'static AuthenticationConfig, + latency_timer: &mut LatencyTimer, ) -> auth::Result> { use BackendType::*; @@ -245,7 +257,16 @@ impl BackendType<'_, ClientCredentials<'_>> { ); let api = api.as_ref(); - auth_quirks(api, extra, creds, client, allow_cleartext, config).await? + auth_quirks( + api, + extra, + creds, + client, + allow_cleartext, + config, + latency_timer, + ) + .await? } Postgres(api, creds) => { info!( @@ -255,7 +276,16 @@ impl BackendType<'_, ClientCredentials<'_>> { ); let api = api.as_ref(); - auth_quirks(api, extra, creds, client, allow_cleartext, config).await? + auth_quirks( + api, + extra, + creds, + client, + allow_cleartext, + config, + latency_timer, + ) + .await? } // NOTE: this auth backend doesn't use client credentials. Link(url) => { diff --git a/proxy/src/auth/backend/classic.rs b/proxy/src/auth/backend/classic.rs index e7013a4cac..aee0057606 100644 --- a/proxy/src/auth/backend/classic.rs +++ b/proxy/src/auth/backend/classic.rs @@ -4,6 +4,7 @@ use crate::{ compute, config::AuthenticationConfig, console::{self, AuthInfo, ConsoleReqExtra}, + proxy::LatencyTimer, sasl, scram, stream::PqStream, }; @@ -16,6 +17,7 @@ pub(super) async fn authenticate( creds: &ClientCredentials<'_>, client: &mut PqStream, config: &'static AuthenticationConfig, + latency_timer: &mut LatencyTimer, ) -> auth::Result> { info!("fetching user's authentication info"); let info = api.get_auth_info(extra, creds).await?.unwrap_or_else(|| { @@ -36,24 +38,26 @@ pub(super) async fn authenticate( info!("auth endpoint chooses SCRAM"); let scram = auth::Scram(&secret); - let auth_flow = flow.begin(scram).await.map_err(|error| { - warn!(?error, "error sending scram acknowledgement"); - error - })?; - let auth_outcome = tokio::time::timeout( config.scram_protocol_timeout, - auth_flow.authenticate(), + async { + // pause the timer while we communicate with the client + let _paused = latency_timer.pause(); + + flow.begin(scram).await.map_err(|error| { + warn!(?error, "error sending scram acknowledgement"); + error + })?.authenticate().await.map_err(|error| { + warn!(?error, "error processing scram messages"); + error + }) + } ) .await .map_err(|error| { warn!("error processing scram messages error = authentication timed out, execution time exeeded {} seconds", config.scram_protocol_timeout.as_secs()); auth::io::Error::new(auth::io::ErrorKind::TimedOut, error) - })? - .map_err(|error| { - warn!(?error, "error processing scram messages"); - error - })?; + })??; let client_key = match auth_outcome { sasl::Outcome::Success(key) => key, diff --git a/proxy/src/auth/backend/hacks.rs b/proxy/src/auth/backend/hacks.rs index 53958bf337..895683af1b 100644 --- a/proxy/src/auth/backend/hacks.rs +++ b/proxy/src/auth/backend/hacks.rs @@ -1,6 +1,7 @@ use super::{AuthSuccess, ComputeCredentials}; use crate::{ auth::{self, AuthFlow, ClientCredentials}, + proxy::LatencyTimer, stream, }; use tokio::io::{AsyncRead, AsyncWrite}; @@ -12,8 +13,13 @@ use tracing::{info, warn}; /// use this mechanism for websocket connections. pub async fn cleartext_hack( client: &mut stream::PqStream, + latency_timer: &mut LatencyTimer, ) -> auth::Result> { warn!("cleartext auth flow override is enabled, proceeding"); + + // pause the timer while we communicate with the client + let _paused = latency_timer.pause(); + let password = AuthFlow::new(client) .begin(auth::CleartextPassword) .await? @@ -32,8 +38,13 @@ pub async fn cleartext_hack( pub async fn password_hack( creds: &mut ClientCredentials<'_>, client: &mut stream::PqStream, + latency_timer: &mut LatencyTimer, ) -> auth::Result> { warn!("project not specified, resorting to the password hack auth flow"); + + // pause the timer while we communicate with the client + let _paused = latency_timer.pause(); + let payload = AuthFlow::new(client) .begin(auth::PasswordHack) .await? diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index 9a01a27fc4..884aae1651 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -106,17 +106,26 @@ static COMPUTE_CONNECTION_LATENCY: Lazy = Lazy::new(|| { }); pub struct LatencyTimer { - start: Instant, + // time since the stopwatch was started + start: Option, + // accumulated time on the stopwatch + accumulated: std::time::Duration, + // label data protocol: &'static str, cache_miss: bool, pool_miss: bool, outcome: &'static str, } +pub struct LatencyTimerPause<'a> { + timer: &'a mut LatencyTimer, +} + impl LatencyTimer { pub fn new(protocol: &'static str) -> Self { Self { - start: Instant::now(), + start: Some(Instant::now()), + accumulated: std::time::Duration::ZERO, protocol, cache_miss: false, // by default we don't do pooling @@ -126,6 +135,13 @@ impl LatencyTimer { } } + pub fn pause(&mut self) -> LatencyTimerPause<'_> { + // stop the stopwatch and record the time that we have accumulated + let start = self.start.take().expect("latency timer should be started"); + self.accumulated += start.elapsed(); + LatencyTimerPause { timer: self } + } + pub fn cache_miss(&mut self) { self.cache_miss = true; } @@ -139,9 +155,17 @@ impl LatencyTimer { } } +impl Drop for LatencyTimerPause<'_> { + fn drop(&mut self) { + // start the stopwatch again + self.timer.start = Some(Instant::now()); + } +} + impl Drop for LatencyTimer { fn drop(&mut self) { - let duration = self.start.elapsed().as_secs_f64(); + let duration = + self.start.map(|start| start.elapsed()).unwrap_or_default() + self.accumulated; COMPUTE_CONNECTION_LATENCY .with_label_values(&[ self.protocol, @@ -149,7 +173,7 @@ impl Drop for LatencyTimer { bool_to_str(self.pool_miss), self.outcome, ]) - .observe(duration) + .observe(duration.as_secs_f64()) } } @@ -862,10 +886,16 @@ impl Client<'_, S> { application_name: params.get("application_name"), }; - let latency_timer = LatencyTimer::new(mode.protocol_label()); + let mut latency_timer = LatencyTimer::new(mode.protocol_label()); let auth_result = match creds - .authenticate(&extra, &mut stream, mode.allow_cleartext(), config) + .authenticate( + &extra, + &mut stream, + mode.allow_cleartext(), + config, + &mut latency_timer, + ) .await { Ok(auth_result) => auth_result, From 2bd79906d944fa1e670727196e613192fc994c58 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Fri, 27 Oct 2023 21:09:34 +0300 Subject: [PATCH 40/69] fix: possible page_service hang on cancel (#5696) Fixes #5341, one more suspected case, see: https://github.com/neondatabase/neon/issues/5341#issuecomment-1783052379 - races `MaybeWriteOnly::shutdown` with cancellation - switches to using `AsyncWriteExt::write_buf` - notes cancellation safety for shutdown --- libs/postgres_backend/src/lib.rs | 21 ++++++++++++++++----- libs/pq_proto/src/framed.rs | 9 +++------ 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/libs/postgres_backend/src/lib.rs b/libs/postgres_backend/src/lib.rs index 8ce7f85481..ad3af4e794 100644 --- a/libs/postgres_backend/src/lib.rs +++ b/libs/postgres_backend/src/lib.rs @@ -242,6 +242,7 @@ impl MaybeWriteOnly { } } + /// Cancellation safe as long as the underlying IO is cancellation safe. async fn shutdown(&mut self) -> io::Result<()> { match self { MaybeWriteOnly::Full(framed) => framed.shutdown().await, @@ -393,13 +394,23 @@ impl PostgresBackend { shutdown_watcher: F, ) -> Result<(), QueryError> where - F: Fn() -> S, + F: Fn() -> S + Clone, S: Future, { - let ret = self.run_message_loop(handler, shutdown_watcher).await; - // socket might be already closed, e.g. if previously received error, - // so ignore result. - self.framed.shutdown().await.ok(); + let ret = self + .run_message_loop(handler, shutdown_watcher.clone()) + .await; + + tokio::select! { + _ = shutdown_watcher() => { + // do nothing; we most likely got already stopped by shutdown and will log it next. + } + _ = self.framed.shutdown() => { + // socket might be already closed, e.g. if previously received error, + // so ignore result. + }, + } + match ret { Ok(()) => Ok(()), Err(QueryError::Shutdown) => { diff --git a/libs/pq_proto/src/framed.rs b/libs/pq_proto/src/framed.rs index c12898a05c..6e97b8c2a0 100644 --- a/libs/pq_proto/src/framed.rs +++ b/libs/pq_proto/src/framed.rs @@ -214,27 +214,24 @@ where } } +/// Cancellation safe as long as the AsyncWrite is cancellation safe. async fn flush( stream: &mut S, write_buf: &mut BytesMut, ) -> Result<(), io::Error> { while write_buf.has_remaining() { - let bytes_written = stream.write(write_buf.chunk()).await?; + let bytes_written = stream.write_buf(write_buf).await?; if bytes_written == 0 { return Err(io::Error::new( ErrorKind::WriteZero, "failed to write message", )); } - // The advanced part will be garbage collected, likely during shifting - // data left on next attempt to write to buffer when free space is not - // enough. - write_buf.advance(bytes_written); } - write_buf.clear(); stream.flush().await } +/// Cancellation safe as long as the AsyncWrite is cancellation safe. async fn shutdown( stream: &mut S, write_buf: &mut BytesMut, From bd59349af373cbd4965c3f8e49facb7623ce5f88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Sat, 28 Oct 2023 03:47:26 +0200 Subject: [PATCH 41/69] Fix Rust 1.74 warnings (#5702) Fixes new warnings and clippy changes introduced by version 1.74 of the rust compiler toolchain. --- pageserver/src/deletion_queue/list_writer.rs | 2 +- .../tenant/remote_timeline_client/index.rs | 2 +- pageserver/src/tenant/timeline.rs | 24 +++++++++---------- .../walreceiver/walreceiver_connection.rs | 6 ++--- pageserver/src/walingest.rs | 4 ++-- proxy/src/config.rs | 2 +- safekeeper/src/timelines_global_map.rs | 2 +- safekeeper/src/wal_storage.rs | 22 ++++++++--------- 8 files changed, 31 insertions(+), 33 deletions(-) diff --git a/pageserver/src/deletion_queue/list_writer.rs b/pageserver/src/deletion_queue/list_writer.rs index 21b8c356cd..6727957b2a 100644 --- a/pageserver/src/deletion_queue/list_writer.rs +++ b/pageserver/src/deletion_queue/list_writer.rs @@ -349,7 +349,7 @@ impl ListWriter { info!("Started deletion frontend worker"); // Synchronous, but we only do it once per process lifetime so it's tolerable - if let Err(e) = create_dir_all(&self.conf.deletion_prefix()) { + if let Err(e) = create_dir_all(self.conf.deletion_prefix()) { tracing::error!( "Failed to create deletion list directory {}, deletions will not be executed ({e})", self.conf.deletion_prefix(), diff --git a/pageserver/src/tenant/remote_timeline_client/index.rs b/pageserver/src/tenant/remote_timeline_client/index.rs index 05f9f5dcd2..7cf963ca9d 100644 --- a/pageserver/src/tenant/remote_timeline_client/index.rs +++ b/pageserver/src/tenant/remote_timeline_client/index.rs @@ -98,7 +98,7 @@ impl IndexPart { const LATEST_VERSION: usize = 4; // Versions we may see when reading from a bucket. - pub const KNOWN_VERSIONS: &[usize] = &[1, 2, 3, 4]; + pub const KNOWN_VERSIONS: &'static [usize] = &[1, 2, 3, 4]; pub const FILE_NAME: &'static str = "index_part.json"; diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 278290eef4..4608683c4a 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -1859,19 +1859,17 @@ impl Timeline { "aborted because task_mgr shutdown requested".to_string() }; - loop { - tokio::select! { - res = &mut calculation => { return res } - reason = timeline_state_cancellation => { - debug!(reason = reason, "cancelling calculation"); - cancel.cancel(); - return calculation.await; - } - reason = taskmgr_shutdown_cancellation => { - debug!(reason = reason, "cancelling calculation"); - cancel.cancel(); - return calculation.await; - } + tokio::select! { + res = &mut calculation => { res } + reason = timeline_state_cancellation => { + debug!(reason = reason, "cancelling calculation"); + cancel.cancel(); + calculation.await + } + reason = taskmgr_shutdown_cancellation => { + debug!(reason = reason, "cancelling calculation"); + cancel.cancel(); + calculation.await } } } diff --git a/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs index 0831b9ceda..3a6599ec55 100644 --- a/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs +++ b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs @@ -122,7 +122,7 @@ pub(super) async fn handle_walreceiver_connection( // Connect to the database in replication mode. info!("connecting to {wal_source_connconf:?}"); - let (mut replication_client, connection) = { + let (replication_client, connection) = { let mut config = wal_source_connconf.to_tokio_postgres_config(); config.application_name("pageserver"); config.replication_mode(tokio_postgres::config::ReplicationMode::Physical); @@ -205,7 +205,7 @@ pub(super) async fn handle_walreceiver_connection( gauge.dec(); } - let identify = identify_system(&mut replication_client).await?; + let identify = identify_system(&replication_client).await?; info!("{identify:?}"); let end_of_wal = Lsn::from(u64::from(identify.xlogpos)); @@ -444,7 +444,7 @@ struct IdentifySystem { struct IdentifyError; /// Run the postgres `IDENTIFY_SYSTEM` command -async fn identify_system(client: &mut Client) -> anyhow::Result { +async fn identify_system(client: &Client) -> anyhow::Result { let query_str = "IDENTIFY_SYSTEM"; let response = client.simple_query(query_str).await?; diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index fb1dbcd6ba..23367928d3 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -443,7 +443,7 @@ impl<'a> WalIngest<'a> { &mut self, buf: &mut Bytes, modification: &mut DatadirModification<'_>, - decoded: &mut DecodedWALRecord, + decoded: &DecodedWALRecord, ctx: &RequestContext, ) -> anyhow::Result<()> { // Handle VM bit updates that are implicitly part of heap records. @@ -749,7 +749,7 @@ impl<'a> WalIngest<'a> { &mut self, buf: &mut Bytes, modification: &mut DatadirModification<'_>, - decoded: &mut DecodedWALRecord, + decoded: &DecodedWALRecord, ctx: &RequestContext, ) -> anyhow::Result<()> { // Handle VM bit updates that are implicitly part of heap records. diff --git a/proxy/src/config.rs b/proxy/src/config.rs index 96856ba051..9607ecd153 100644 --- a/proxy/src/config.rs +++ b/proxy/src/config.rs @@ -223,7 +223,7 @@ pub struct CacheOptions { impl CacheOptions { /// Default options for [`crate::console::provider::NodeInfoCache`]. - pub const DEFAULT_OPTIONS_NODE_INFO: &str = "size=4000,ttl=4m"; + pub const DEFAULT_OPTIONS_NODE_INFO: &'static str = "size=4000,ttl=4m"; /// Parse cache options passed via cmdline. /// Example: [`Self::DEFAULT_OPTIONS_NODE_INFO`]. diff --git a/safekeeper/src/timelines_global_map.rs b/safekeeper/src/timelines_global_map.rs index 1434644123..cbb3342e40 100644 --- a/safekeeper/src/timelines_global_map.rs +++ b/safekeeper/src/timelines_global_map.rs @@ -296,8 +296,8 @@ impl GlobalTimelines { global_lock .timelines .values() - .cloned() .filter(|t| !t.is_cancelled()) + .cloned() .collect() } diff --git a/safekeeper/src/wal_storage.rs b/safekeeper/src/wal_storage.rs index 2070122e8e..fa44b24258 100644 --- a/safekeeper/src/wal_storage.rs +++ b/safekeeper/src/wal_storage.rs @@ -188,7 +188,7 @@ impl PhysicalStorage { } /// Call fdatasync if config requires so. - async fn fdatasync_file(&mut self, file: &mut File) -> Result<()> { + async fn fdatasync_file(&mut self, file: &File) -> Result<()> { if !self.conf.no_sync { self.metrics .observe_flush_seconds(time_io_closure(file.sync_data()).await?); @@ -197,7 +197,7 @@ impl PhysicalStorage { } /// Call fsync if config requires so. - async fn fsync_file(&mut self, file: &mut File) -> Result<()> { + async fn fsync_file(&mut self, file: &File) -> Result<()> { if !self.conf.no_sync { self.metrics .observe_flush_seconds(time_io_closure(file.sync_all()).await?); @@ -231,7 +231,7 @@ impl PhysicalStorage { .with_context(|| format!("Failed to open log file {:?}", &wal_file_path))?; write_zeroes(&mut file, self.wal_seg_size).await?; - self.fsync_file(&mut file).await?; + self.fsync_file(&file).await?; Ok((file, true)) } } @@ -255,7 +255,7 @@ impl PhysicalStorage { if xlogoff + buf.len() == self.wal_seg_size { // If we reached the end of a WAL segment, flush and close it. - self.fdatasync_file(&mut file).await?; + self.fdatasync_file(&file).await?; // Rename partial file to completed file let (wal_file_path, wal_file_partial_path) = @@ -277,8 +277,8 @@ impl PhysicalStorage { async fn write_exact(&mut self, pos: Lsn, mut buf: &[u8]) -> Result<()> { if self.write_lsn != pos { // need to flush the file before discarding it - if let Some(mut file) = self.file.take() { - self.fdatasync_file(&mut file).await?; + if let Some(file) = self.file.take() { + self.fdatasync_file(&file).await?; } self.write_lsn = pos; @@ -367,8 +367,8 @@ impl Storage for PhysicalStorage { return Ok(()); } - if let Some(mut unflushed_file) = self.file.take() { - self.fdatasync_file(&mut unflushed_file).await?; + if let Some(unflushed_file) = self.file.take() { + self.fdatasync_file(&unflushed_file).await?; self.file = Some(unflushed_file); } else { // We have unflushed data (write_lsn != flush_lsn), but no file. @@ -410,8 +410,8 @@ impl Storage for PhysicalStorage { } // Close previously opened file, if any - if let Some(mut unflushed_file) = self.file.take() { - self.fdatasync_file(&mut unflushed_file).await?; + if let Some(unflushed_file) = self.file.take() { + self.fdatasync_file(&unflushed_file).await?; } let xlogoff = end_pos.segment_offset(self.wal_seg_size); @@ -425,7 +425,7 @@ impl Storage for PhysicalStorage { // Fill end with zeroes file.seek(SeekFrom::Start(xlogoff as u64)).await?; write_zeroes(&mut file, self.wal_seg_size - xlogoff).await?; - self.fdatasync_file(&mut file).await?; + self.fdatasync_file(&file).await?; if !is_partial { // Make segment partial once again From 964c5c56b75ac98344af6c364a2764864f480cfd Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Mon, 30 Oct 2023 08:38:56 +0000 Subject: [PATCH 42/69] proxy: dont retry server errors (#5694) ## Problem accidental spam ## Summary of changes don't spam control plane if control plane is down :) ## Checklist before requesting a review - [x] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. ## Checklist before merging - [ ] Do not forget to reformat commit message to not include the above checklist --- proxy/src/console/provider.rs | 2 -- proxy/src/proxy/tests.rs | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/proxy/src/console/provider.rs b/proxy/src/console/provider.rs index 86fd810ac3..32c3467092 100644 --- a/proxy/src/console/provider.rs +++ b/proxy/src/console/provider.rs @@ -97,8 +97,6 @@ pub mod errors { !text.contains("quota exceeded") && !text.contains("the limit for current plan reached") } - // retry server errors - Self::Console { status, .. } if status.is_server_error() => true, _ => false, } } diff --git a/proxy/src/proxy/tests.rs b/proxy/src/proxy/tests.rs index 8f7f461bef..142c32fb84 100644 --- a/proxy/src/proxy/tests.rs +++ b/proxy/src/proxy/tests.rs @@ -409,7 +409,7 @@ impl TestBackend for TestConnectMechanism { } ConnectAction::WakeRetry => { let err = console::errors::ApiError::Console { - status: http::StatusCode::INTERNAL_SERVER_ERROR, + status: http::StatusCode::BAD_REQUEST, text: "TEST".into(), }; assert!(err.could_retry()); From 87db4b441c31fb704772beddf46307bea444bbe9 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 30 Oct 2023 09:18:40 +0000 Subject: [PATCH 43/69] pageserver: cleaner shutdown in timeline delete (#5701) The flush task logs a backtrace if it tries to upload and remote timeline client is already in stopped state. Therefore we cannot shut them down concurrently: flush task must be shut down first. This wasn't more obvious because: - Timeline deletions IRL usually happen when not much is being written - In tests, there is a global allow-list for this log It's not obvious whether removing the global log allow list is safe, this PR was prompted by how the log spam got in my way when testing deletion changes. --- pageserver/src/tenant/timeline/delete.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pageserver/src/tenant/timeline/delete.rs b/pageserver/src/tenant/timeline/delete.rs index 8cead5ebf2..6d30664515 100644 --- a/pageserver/src/tenant/timeline/delete.rs +++ b/pageserver/src/tenant/timeline/delete.rs @@ -38,6 +38,14 @@ async fn stop_tasks(timeline: &Timeline) -> Result<(), DeleteTimelineError> { } debug!("wal receiver shutdown confirmed"); + // Shut down the layer flush task before the remote client, as one depends on the other + task_mgr::shutdown_tasks( + Some(TaskKind::LayerFlushTask), + Some(timeline.tenant_id), + Some(timeline.timeline_id), + ) + .await; + // Prevent new uploads from starting. if let Some(remote_client) = timeline.remote_client.as_ref() { let res = remote_client.stop(); From 9cdffd164a8b3d486360fd306bb37ac8fb5ebed3 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 30 Oct 2023 12:06:08 +0200 Subject: [PATCH 44/69] Prevent SIGSEGV in apply_error_callback when record was not decoded (#5703) ## Problem See https://neondb.slack.com/archives/C036U0GRMRB/p1698652221399419?thread_ts=1698438997.903919&cid=C036U0GRMRB ## Summary of changes Check if record pointer is not NULL before trying to print record descriptor ## Checklist before requesting a review - [ ] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. ## Checklist before merging - [ ] Do not forget to reformat commit message to not include the above checklist Co-authored-by: Konstantin Knizhnik --- pgxn/neon_walredo/walredoproc.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pgxn/neon_walredo/walredoproc.c b/pgxn/neon_walredo/walredoproc.c index e0c6c6e30a..bdc50b0aa9 100644 --- a/pgxn/neon_walredo/walredoproc.c +++ b/pgxn/neon_walredo/walredoproc.c @@ -895,7 +895,12 @@ apply_error_callback(void *arg) StringInfoData buf; initStringInfo(&buf); - xlog_outdesc(&buf, record); +#if PG_VERSION_NUM >= 150000 + if (record->record) +#else + if (record->decoded_record) +#endif + xlog_outdesc(&buf, record); /* translator: %s is a WAL record description */ errcontext("WAL redo at %X/%X for %s", From 07c2b29895321faeb86b0aa8e73c40381ce68adf Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 30 Oct 2023 13:24:52 +0000 Subject: [PATCH 45/69] pageserver: fix error logging on stray timeline files (#5712) ## Problem If there were stray files in the timelines/ dir after tenant deletion, pageserver could panic on out of range. ## Summary of changes Use iterator `take()`, which doesn't care if the number of elements available is less than requested. --- pageserver/src/tenant/delete.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pageserver/src/tenant/delete.rs b/pageserver/src/tenant/delete.rs index 3e2d6ca7bb..87b48e4bee 100644 --- a/pageserver/src/tenant/delete.rs +++ b/pageserver/src/tenant/delete.rs @@ -148,7 +148,8 @@ async fn ensure_timelines_dir_empty(timelines_path: &Utf8Path) -> Result<(), Del // Assert timelines dir is empty. if !fs_ext::is_directory_empty(timelines_path).await? { // Display first 10 items in directory - let list = &fs_ext::list_dir(timelines_path).await.context("list_dir")?[..10]; + let list = fs_ext::list_dir(timelines_path).await.context("list_dir")?; + let list = &list.into_iter().take(10).collect::>(); return Err(DeleteTenantError::Other(anyhow::anyhow!( "Timelines directory is not empty after all timelines deletion: {list:?}" ))); From 4db8efb2cf16f67bd8ecf23dd71f54de506cde50 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Mon, 30 Oct 2023 16:21:30 +0200 Subject: [PATCH 46/69] Layer: logging fixes (#5676) - include Layer generation in the default display, with Generation::Broken as `-broken` - omit layer from `layer_gc` span because the api it works with needs to support N layers, so the api needs to log each layer --- pageserver/src/tenant/storage_layer/layer.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index 759f6d664c..5a26421b5b 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -47,7 +47,16 @@ pub(crate) struct Layer(Arc); impl std::fmt::Display for Layer { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.layer_desc().short_id()) + if matches!(self.0.generation, Generation::Broken) { + write!(f, "{}-broken", self.layer_desc().short_id()) + } else { + write!( + f, + "{}{}", + self.layer_desc().short_id(), + self.0.generation.get_suffix() + ) + } } } @@ -433,7 +442,7 @@ impl Drop for LayerInner { return; } - let span = tracing::info_span!(parent: None, "layer_gc", tenant_id = %self.layer_desc().tenant_id, timeline_id = %self.layer_desc().timeline_id, layer = %self); + let span = tracing::info_span!(parent: None, "layer_gc", tenant_id = %self.layer_desc().tenant_id, timeline_id = %self.layer_desc().timeline_id); let path = std::mem::take(&mut self.path); let file_name = self.layer_desc().filename(); From e675f4cec85b9341448230af72ef732fb0dae02e Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 30 Oct 2023 15:18:32 +0000 Subject: [PATCH 47/69] =?UTF-8?q?tests:=20fix=20missing=20comma=20=20in=20?= =?UTF-8?q?test=5Ftimeline=5Fdeletion=5Fwith=5Ffiles=5Fstuck=5F=E2=80=A6?= =?UTF-8?q?=20(#5713)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …in_upload_queue This was a syntax mistake in https://github.com/neondatabase/neon/pull/5149 We didn't notice because the situation the log allow list covers is a relative rare race. --- test_runner/regress/test_remote_storage.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py index f201c735e1..31bc97703e 100644 --- a/test_runner/regress/test_remote_storage.py +++ b/test_runner/regress/test_remote_storage.py @@ -585,9 +585,11 @@ def test_timeline_deletion_with_files_stuck_in_upload_queue( # checkpoint operations. Hence, checkpoint is allowed to fail now. log.info("sending delete request") checkpoint_allowed_to_fail.set() - env.pageserver.allowed_errors.append( - ".* ERROR .*Error processing HTTP request: InternalServerError\\(The timeline or pageserver is shutting down" - ".* ERROR .*[Cc]ould not flush frozen layer.*" + env.pageserver.allowed_errors.extend( + [ + ".* ERROR .*Error processing HTTP request: InternalServerError\\(The timeline or pageserver is shutting down", + ".* ERROR .*[Cc]ould not flush frozen layer.*", + ] ) # Generous timeout, because currently deletions can get blocked waiting for compaction From ad99fa5f0393e2679e5323df653c508ffa0ac072 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 30 Oct 2023 17:29:25 +0200 Subject: [PATCH 48/69] Grant BYPASSRLS and REPLICATION to exited roles (#5657) ## Problem Role need to have REPLICATION privilege to be able to used for logical replication. New roles are created with this option. This PR tries to update existed roles. ## Summary of changes Update roles in `handle_roles` method ## Checklist before requesting a review - [ ] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. ## Checklist before merging - [ ] Do not forget to reformat commit message to not include the above checklist --------- Co-authored-by: Konstantin Knizhnik --- compute_tools/src/pg_helpers.rs | 7 ++++++- compute_tools/src/spec.rs | 5 ++++- libs/compute_api/src/spec.rs | 2 ++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/compute_tools/src/pg_helpers.rs b/compute_tools/src/pg_helpers.rs index b79e516650..8722822f5e 100644 --- a/compute_tools/src/pg_helpers.rs +++ b/compute_tools/src/pg_helpers.rs @@ -193,11 +193,16 @@ impl Escaping for PgIdent { /// Build a list of existing Postgres roles pub fn get_existing_roles(xact: &mut Transaction<'_>) -> Result> { let postgres_roles = xact - .query("SELECT rolname, rolpassword FROM pg_catalog.pg_authid", &[])? + .query( + "SELECT rolname, rolpassword, rolreplication, rolbypassrls FROM pg_catalog.pg_authid", + &[], + )? .iter() .map(|row| Role { name: row.get("rolname"), encrypted_password: row.get("rolpassword"), + replication: Some(row.get("rolreplication")), + bypassrls: Some(row.get("rolbypassrls")), options: None, }) .collect(); diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index f7ca2eb33c..6e4d4ccf49 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -265,6 +265,8 @@ pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> { let action = if let Some(r) = pg_role { if (r.encrypted_password.is_none() && role.encrypted_password.is_some()) || (r.encrypted_password.is_some() && role.encrypted_password.is_none()) + || !r.bypassrls.unwrap_or(false) + || !r.replication.unwrap_or(false) { RoleAction::Update } else if let Some(pg_pwd) = &r.encrypted_password { @@ -296,7 +298,8 @@ pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> { match action { RoleAction::None => {} RoleAction::Update => { - let mut query: String = format!("ALTER ROLE {} ", name.pg_quote()); + let mut query: String = + format!("ALTER ROLE {} BYPASSRLS REPLICATION", name.pg_quote()); query.push_str(&role.to_pg_options()); xact.execute(query.as_str(), &[])?; } diff --git a/libs/compute_api/src/spec.rs b/libs/compute_api/src/spec.rs index cfbd50d38a..c16deceebb 100644 --- a/libs/compute_api/src/spec.rs +++ b/libs/compute_api/src/spec.rs @@ -190,6 +190,8 @@ pub struct DeltaOp { pub struct Role { pub name: PgIdent, pub encrypted_password: Option, + pub replication: Option, + pub bypassrls: Option, pub options: GenericOptions, } From d8c21ec70d60f5e4a4675a16bc596cbf60eefc8f Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Mon, 30 Oct 2023 16:43:06 +0000 Subject: [PATCH 49/69] fix nightly 1.75 (#5719) ## Problem Neon doesn't compile on nightly and had numerous clippy complaints. ## Summary of changes 1. Fixed troublesome dependency 2. Fixed or ignored the lints where appropriate --- Cargo.lock | 23 ++++--------------- libs/postgres_ffi/wal_craft/src/lib.rs | 1 + .../walreceiver/walreceiver_connection.rs | 2 +- proxy/src/console/provider/mock.rs | 2 +- proxy/src/scram.rs | 1 - 5 files changed, 7 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f7598a79cf..39c33304ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1609,16 +1609,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "ctor" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" -dependencies = [ - "quote", - "syn 1.0.109", -] - [[package]] name = "ctr" version = "0.6.0" @@ -2714,11 +2704,10 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" dependencies = [ - "cfg-if", "value-bag", ] @@ -6011,13 +6000,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "value-bag" -version = "1.0.0-alpha.9" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55" -dependencies = [ - "ctor", - "version_check", -] +checksum = "4a72e1902dde2bd6441347de2b70b7f5d59bf157c6c62f0c44572607a1d55bbe" [[package]] name = "vcpkg" diff --git a/libs/postgres_ffi/wal_craft/src/lib.rs b/libs/postgres_ffi/wal_craft/src/lib.rs index 75ffd3f055..281a180e3b 100644 --- a/libs/postgres_ffi/wal_craft/src/lib.rs +++ b/libs/postgres_ffi/wal_craft/src/lib.rs @@ -14,6 +14,7 @@ macro_rules! xlog_utils_test { ($version:ident) => { #[path = "."] mod $version { + #[allow(unused_imports)] pub use postgres_ffi::$version::wal_craft_test_export::*; #[allow(clippy::duplicate_mod)] #[cfg(test)] diff --git a/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs index 3a6599ec55..3e56753ad4 100644 --- a/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs +++ b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs @@ -459,7 +459,7 @@ async fn identify_system(client: &Client) -> anyhow::Result { // extract the row contents into an IdentifySystem struct. // written as a closure so I can use ? for Option here. - if let Some(SimpleQueryMessage::Row(first_row)) = response.get(0) { + if let Some(SimpleQueryMessage::Row(first_row)) = response.first() { Ok(IdentifySystem { systemid: get_parse(first_row, 0)?, timeline: get_parse(first_row, 1)?, diff --git a/proxy/src/console/provider/mock.rs b/proxy/src/console/provider/mock.rs index 282567269d..750a2d141e 100644 --- a/proxy/src/console/provider/mock.rs +++ b/proxy/src/console/provider/mock.rs @@ -59,7 +59,7 @@ impl Api { let rows = client.query(query, &[&creds.user]).await?; // We can get at most one row, because `rolname` is unique. - let row = match rows.get(0) { + let row = match rows.first() { Some(row) => row, // This means that the user doesn't exist, so there can be no secret. // However, this is still a *valid* outcome which is very similar diff --git a/proxy/src/scram.rs b/proxy/src/scram.rs index 07822e8da5..2de26af96b 100644 --- a/proxy/src/scram.rs +++ b/proxy/src/scram.rs @@ -18,7 +18,6 @@ mod password; pub use exchange::Exchange; pub use key::ScramKey; pub use secret::ServerSecret; -pub use secret::*; use hmac::{Hmac, Mac}; use sha2::{Digest, Sha256}; From 9c35e1e6e5609780a5bc55b5e98a4a711f6e6465 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 30 Oct 2023 18:32:30 +0000 Subject: [PATCH 50/69] pageserver: downgrade slow task warnings from warn to info (#5724) ## Problem In #5658 we suppressed the first-iteration output from these logs, but the volume of warnings is still problematic. ## Summary of changes - Downgrade all slow task warnings to INFO. The information is still there if we actively want to know about which tasks are running slowly, without polluting the overall stream of warnings with situations that are unsurprising to us. - Revert the previous change so that we output on the first iteration as we used to do. There is no reason to suppress these, now that the severity is just info. --- pageserver/src/tenant/tasks.rs | 38 ++++++++++++---------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/pageserver/src/tenant/tasks.rs b/pageserver/src/tenant/tasks.rs index f419d2e73a..eb77f7c83a 100644 --- a/pageserver/src/tenant/tasks.rs +++ b/pageserver/src/tenant/tasks.rs @@ -162,8 +162,11 @@ async fn compaction_loop(tenant: Arc, cancel: CancellationToken) { // TODO: we shouldn't need to await to find tenant and this could be moved outside of // loop, #3501. There are also additional "allowed_errors" in tests. - if first && random_init_delay(period, &cancel).await.is_err() { - break; + if first { + first = false; + if random_init_delay(period, &cancel).await.is_err() { + break; + } } let started_at = Instant::now(); @@ -193,16 +196,7 @@ async fn compaction_loop(tenant: Arc, cancel: CancellationToken) { } }; - if !first { - // The first iteration is typically much slower, because all tenants compete for the - // compaction sempahore to run, and because of concurrent startup work like initializing - // logical sizes. To avoid routinely spamming warnings, we suppress this log on first iteration. - warn_when_period_overrun( - started_at.elapsed(), - period, - BackgroundLoopKind::Compaction, - ); - } + warn_when_period_overrun(started_at.elapsed(), period, BackgroundLoopKind::Compaction); // Sleep if tokio::time::timeout(sleep_duration, cancel.cancelled()) @@ -211,8 +205,6 @@ async fn compaction_loop(tenant: Arc, cancel: CancellationToken) { { break; } - - first = false; } } .await; @@ -247,8 +239,11 @@ async fn gc_loop(tenant: Arc, cancel: CancellationToken) { let period = tenant.get_gc_period(); - if first && random_init_delay(period, &cancel).await.is_err() { - break; + if first { + first = false; + if random_init_delay(period, &cancel).await.is_err() { + break; + } } let started_at = Instant::now(); @@ -282,12 +277,7 @@ async fn gc_loop(tenant: Arc, cancel: CancellationToken) { } }; - if !first { - // The first iteration is typically much slower, because all tenants compete for the - // compaction sempahore to run, and because of concurrent startup work like initializing - // logical sizes. To avoid routinely spamming warnings, we suppress this log on first iteration. - warn_when_period_overrun(started_at.elapsed(), period, BackgroundLoopKind::Gc); - } + warn_when_period_overrun(started_at.elapsed(), period, BackgroundLoopKind::Gc); // Sleep if tokio::time::timeout(sleep_duration, cancel.cancelled()) @@ -296,8 +286,6 @@ async fn gc_loop(tenant: Arc, cancel: CancellationToken) { { break; } - - first = false; } } .await; @@ -373,7 +361,7 @@ pub(crate) fn warn_when_period_overrun( // humantime does no significant digits clamping whereas Duration's debug is a bit more // intelligent. however it makes sense to keep the "configuration format" for period, even // though there's no way to output the actual config value. - warn!( + info!( ?elapsed, period = %humantime::format_duration(period), ?task, From 1588601503a0dc6a92972fd9946d924d5eb1747a Mon Sep 17 00:00:00 2001 From: Shany Pozin Date: Mon, 30 Oct 2023 23:10:24 +0200 Subject: [PATCH 51/69] Move release PR creation to Friday (#5721) Prepare for a new release workflow * Release PR is created on Fridays * The discussion/approval happens during Friday * Sunday morning the deployment will be done in central-il and perf tests will be run * On Monday early IST morning gradually start rolling (starting from US regions as they are still in weekend time) See slack for discussion: https://neondb.slack.com/archives/C04P81J55LK/p1698565305607839?thread_ts=1698428241.031979&cid=C04P81J55LK --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 36af98f96e..3d6fc75e08 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,7 +2,7 @@ name: Create Release Branch on: schedule: - - cron: '0 7 * * 2' + - cron: '0 7 * * 5' workflow_dispatch: jobs: From 7ebe9ca1aca6c6fb23507f984950cfd061e637f0 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 31 Oct 2023 09:32:58 +0100 Subject: [PATCH 52/69] pageserver: `/attach`: clarify semantics of 409 (#5698) context: https://app.incident.io/neondb/incidents/75 specifically: https://neondb.slack.com/archives/C0634NXQ6E7/p1698422852902959?thread_ts=1698419362.155059&cid=C0634NXQ6E7 --- pageserver/src/http/openapi_spec.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index 2d954f6823..d31e4a1554 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -569,7 +569,17 @@ paths: schema: $ref: "#/components/schemas/NotFoundError" "409": - description: Tenant download is already in progress + description: | + The tenant is already known to Pageserver in some way, + and hence this `/attach` call has been rejected. + + Some examples of how this can happen: + - tenant was created on this pageserver + - tenant attachment was started by an earlier call to `/attach`. + + Callers should poll the tenant status's `attachment_status` field, + like for status 202. See the longer description for `POST /attach` + for details. content: application/json: schema: From e5c81fef86b47d319e98e24a8d3693d9659ada21 Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 31 Oct 2023 11:44:35 +0000 Subject: [PATCH 53/69] tests: minor improvements (#5674) Minor changes from while I have been working on HA tests: - Manual pytest executions came with some warnings from `log.warn()` usage - When something fails in a generations-enabled test, it it useful to have a log from the attachment service of what attached when, and with which generation. --------- Co-authored-by: Joonas Koivunen --- control_plane/src/bin/attachment_service.rs | 26 +++++++++++++++++---- test_runner/fixtures/pageserver/utils.py | 4 ++-- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/control_plane/src/bin/attachment_service.rs b/control_plane/src/bin/attachment_service.rs index 0ccd9e75fc..3205703c7d 100644 --- a/control_plane/src/bin/attachment_service.rs +++ b/control_plane/src/bin/attachment_service.rs @@ -12,6 +12,7 @@ use hyper::{Body, Request, Response}; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; use std::{collections::HashMap, sync::Arc}; +use utils::http::endpoint::request_span; use utils::logging::{self, LogFormat}; use utils::signals::{ShutdownSignals, Signal}; @@ -221,8 +222,25 @@ async fn handle_attach_hook(mut req: Request) -> Result, Ap generation: 0, }); - if attach_req.node_id.is_some() { + if let Some(attaching_pageserver) = attach_req.node_id.as_ref() { tenant_state.generation += 1; + tracing::info!( + tenant_id = %attach_req.tenant_id, + ps_id = %attaching_pageserver, + generation = %tenant_state.generation, + "issuing", + ); + } else if let Some(ps_id) = tenant_state.pageserver { + tracing::info!( + tenant_id = %attach_req.tenant_id, + %ps_id, + generation = %tenant_state.generation, + "dropping", + ); + } else { + tracing::info!( + tenant_id = %attach_req.tenant_id, + "no-op: tenant already has no pageserver"); } tenant_state.pageserver = attach_req.node_id; let generation = tenant_state.generation; @@ -240,9 +258,9 @@ async fn handle_attach_hook(mut req: Request) -> Result, Ap fn make_router(persistent_state: PersistentState) -> RouterBuilder { endpoint::make_router() .data(Arc::new(State::new(persistent_state))) - .post("/re-attach", handle_re_attach) - .post("/validate", handle_validate) - .post("/attach-hook", handle_attach_hook) + .post("/re-attach", |r| request_span(r, handle_re_attach)) + .post("/validate", |r| request_span(r, handle_validate)) + .post("/attach-hook", |r| request_span(r, handle_attach_hook)) } #[tokio::main] diff --git a/test_runner/fixtures/pageserver/utils.py b/test_runner/fixtures/pageserver/utils.py index e54b5408b4..007ff387f4 100644 --- a/test_runner/fixtures/pageserver/utils.py +++ b/test_runner/fixtures/pageserver/utils.py @@ -249,7 +249,7 @@ def assert_prefix_empty(neon_env_builder: "NeonEnvBuilder", prefix: Optional[str # this has been seen in the wild by tests with the below contradicting logging # https://neon-github-public-dev.s3.amazonaws.com/reports/pr-5322/6207777020/index.html#suites/3556ed71f2d69272a7014df6dcb02317/53b5c368b5a68865 # this seems like a mock_s3 issue - log.warn( + log.warning( f"contrading ListObjectsV2 response with KeyCount={keys} and Contents={objects}, CommonPrefixes={common_prefixes}, assuming this means KeyCount=0" ) keys = 0 @@ -257,7 +257,7 @@ def assert_prefix_empty(neon_env_builder: "NeonEnvBuilder", prefix: Optional[str # this has been seen in one case with mock_s3: # https://neon-github-public-dev.s3.amazonaws.com/reports/pr-4938/6000769714/index.html#suites/3556ed71f2d69272a7014df6dcb02317/ca01e4f4d8d9a11f # looking at moto impl, it might be there's a race with common prefix (sub directory) not going away with deletes - log.warn( + log.warning( f"contradicting ListObjectsV2 response with KeyCount={keys} and Contents={objects}, CommonPrefixes={common_prefixes}" ) From 896347f307653446cfd9b3c09f1ec94e283e566a Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Tue, 31 Oct 2023 18:40:08 +0200 Subject: [PATCH 54/69] refactor(layer): remove version checking with atomics (#5742) The `LayerInner::version` never needed to be read in more than one place. Clarified while fixing #5737 of which this is the first step. This decrements possible wrong atomics usage in Layer, but does not really fix anything. --- pageserver/src/tenant/storage_layer/layer.rs | 121 ++++++++++--------- 1 file changed, 66 insertions(+), 55 deletions(-) diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index 5a26421b5b..aaba9bd933 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -125,6 +125,7 @@ impl Layer { let inner = Arc::new(DownloadedLayer { owner: owner.clone(), kind: tokio::sync::OnceCell::default(), + version: 0, }); resident = Some(inner.clone()); @@ -163,6 +164,7 @@ impl Layer { let inner = Arc::new(DownloadedLayer { owner: owner.clone(), kind: tokio::sync::OnceCell::default(), + version: 0, }); resident = Some(inner.clone()); let access_stats = LayerAccessStats::empty_will_record_residence_event_later(); @@ -328,16 +330,17 @@ impl Layer { /// read with [`Layer::get_value_reconstruct_data`]. /// /// [`LayerMap::search`]: crate::tenant::layer_map::LayerMap::search +#[derive(Debug)] enum ResidentOrWantedEvicted { Resident(Arc), - WantedEvicted(Weak), + WantedEvicted(Weak, usize), } impl ResidentOrWantedEvicted { fn get(&self) -> Option> { match self { ResidentOrWantedEvicted::Resident(strong) => Some(strong.clone()), - ResidentOrWantedEvicted::WantedEvicted(weak) => match weak.upgrade() { + ResidentOrWantedEvicted::WantedEvicted(weak, _) => match weak.upgrade() { Some(strong) => { LAYER_IMPL_METRICS.inc_raced_wanted_evicted_accesses(); Some(strong) @@ -349,21 +352,16 @@ impl ResidentOrWantedEvicted { /// When eviction is first requested, drop down to holding a [`Weak`]. /// /// Returns `true` if this was the first time eviction was requested. - fn downgrade(&mut self) -> &Weak { - let _was_first = match self { + fn downgrade(&mut self) -> bool { + match self { ResidentOrWantedEvicted::Resident(strong) => { let weak = Arc::downgrade(strong); - *self = ResidentOrWantedEvicted::WantedEvicted(weak); + *self = ResidentOrWantedEvicted::WantedEvicted(weak, strong.version); // returning the weak is not useful, because the drop could had already ran with // the replacement above, and that will take care of cleaning the Option we are in true } - ResidentOrWantedEvicted::WantedEvicted(_) => false, - }; - - match self { - ResidentOrWantedEvicted::WantedEvicted(ref weak) => weak, - _ => unreachable!("just wrote wanted evicted"), + ResidentOrWantedEvicted::WantedEvicted(..) => false, } } } @@ -398,8 +396,10 @@ struct LayerInner { /// [`LayerInner::on_downloaded_layer_drop`]. wanted_evicted: AtomicBool, - /// Version is to make sure we will in fact only evict a file if no new download has been - /// started. + /// Version is to make sure we will only evict a specific download of a file. + /// + /// Incremented for each download, stored in `DownloadedLayer::version` or + /// `ResidentOrWantedEvicted::WantedEvicted`. version: AtomicUsize, /// Allow subscribing to when the layer actually gets evicted. @@ -515,6 +515,14 @@ impl LayerInner { .timeline_path(&timeline.tenant_id, &timeline.timeline_id) .join(desc.filename().to_string()); + let (inner, version) = if let Some(inner) = downloaded { + let version = inner.version; + let resident = ResidentOrWantedEvicted::Resident(inner); + (heavier_once_cell::OnceCell::new(resident), version) + } else { + (heavier_once_cell::OnceCell::default(), 0) + }; + LayerInner { conf, path, @@ -524,12 +532,8 @@ impl LayerInner { access_stats, wanted_garbage_collected: AtomicBool::new(false), wanted_evicted: AtomicBool::new(false), - inner: if let Some(inner) = downloaded { - heavier_once_cell::OnceCell::new(ResidentOrWantedEvicted::Resident(inner)) - } else { - heavier_once_cell::OnceCell::default() - }, - version: AtomicUsize::new(0), + inner, + version: AtomicUsize::new(version), status: tokio::sync::broadcast::channel(1).0, consecutive_failures: AtomicUsize::new(0), generation, @@ -604,7 +608,7 @@ impl LayerInner { loop { let download = move || async move { // disable any scheduled but not yet running eviction deletions for this - self.version.fetch_add(1, Ordering::Relaxed); + let next_version = 1 + self.version.fetch_add(1, Ordering::Relaxed); // no need to make the evict_and_wait wait for the actual download to complete drop(self.status.send(Status::Downloaded)); @@ -655,6 +659,7 @@ impl LayerInner { let res = Arc::new(DownloadedLayer { owner: Arc::downgrade(self), kind: tokio::sync::OnceCell::default(), + version: next_version, }); self.access_stats.record_residence_event( @@ -896,7 +901,7 @@ impl LayerInner { } /// `DownloadedLayer` is being dropped, so it calls this method. - fn on_downloaded_layer_drop(self: Arc) { + fn on_downloaded_layer_drop(self: Arc, version: usize) { let gc = self.wanted_garbage_collected.load(Ordering::Acquire); let evict = self.wanted_evicted.load(Ordering::Acquire); let can_evict = self.have_remote_client; @@ -904,15 +909,16 @@ impl LayerInner { if gc { // do nothing now, only in LayerInner::drop } else if can_evict && evict { - let version = self.version.load(Ordering::Relaxed); - - let span = tracing::info_span!(parent: None, "layer_evict", tenant_id = %self.desc.tenant_id, timeline_id = %self.desc.timeline_id, layer=%self); + let span = tracing::info_span!(parent: None, "layer_evict", tenant_id = %self.desc.tenant_id, timeline_id = %self.desc.timeline_id, layer=%self, %version); // downgrade for queueing, in case there's a tear down already ongoing we should not // hold it alive. let this = Arc::downgrade(&self); drop(self); + // NOTE: this scope *must* never call `self.inner.get` because evict_and_wait might + // drop while the `self.inner` is being locked, leading to a deadlock. + crate::task_mgr::BACKGROUND_RUNTIME.spawn_blocking(move || { let _g = span.entered(); @@ -922,19 +928,15 @@ impl LayerInner { LAYER_IMPL_METRICS.inc_eviction_cancelled(EvictionCancelled::LayerGone); return; }; - this.evict_blocking(version); + match this.evict_blocking(version) { + Ok(()) => LAYER_IMPL_METRICS.inc_completed_evictions(), + Err(reason) => LAYER_IMPL_METRICS.inc_eviction_cancelled(reason), + } }); } } - fn evict_blocking(&self, version: usize) { - match self.evict_blocking0(version) { - Ok(()) => LAYER_IMPL_METRICS.inc_completed_evictions(), - Err(reason) => LAYER_IMPL_METRICS.inc_eviction_cancelled(reason), - } - } - - fn evict_blocking0(&self, version: usize) -> Result<(), EvictionCancelled> { + fn evict_blocking(&self, only_version: usize) -> Result<(), EvictionCancelled> { // deleted or detached timeline, don't do anything. let Some(timeline) = self.timeline.upgrade() else { return Err(EvictionCancelled::TimelineGone); @@ -945,32 +947,34 @@ impl LayerInner { let _permit = { let maybe_downloaded = self.inner.get(); - if version != self.version.load(Ordering::Relaxed) { - // downloadness-state has advanced, we might no longer be the latest eviction - // work; don't do anything. - // - // this is possible to get to by having: - // - // 1. wanted_evicted.store(true) - // 2. ResidentOrWantedEvicted::downgrade - // 3. DownloadedLayer::drop - // 4. LayerInner::get_or_maybe_download - // 5. LayerInner::evict_blocking - return Err(EvictionCancelled::VersionCheckFailed); - } - - // free the DownloadedLayer allocation - match maybe_downloaded.map(|mut g| g.take_and_deinit()) { - Some((taken, permit)) => { - assert!(matches!(taken, ResidentOrWantedEvicted::WantedEvicted(_))); - permit + let (_weak, permit) = match maybe_downloaded { + Some(mut guard) => { + if let ResidentOrWantedEvicted::WantedEvicted(_weak, version) = &*guard { + if *version == only_version { + guard.take_and_deinit() + } else { + // this was not for us; maybe there's another eviction job + // TODO: does it make any sense to stall here? unique versions do not + // matter, we only want to make sure not to evict a resident, which we + // are not doing. + return Err(EvictionCancelled::VersionCheckFailed); + } + } else { + return Err(EvictionCancelled::AlreadyReinitialized); + } } None => { - unreachable!("we do the version checking for this exact reason") + // already deinitialized, perhaps get_or_maybe_download did this and is + // currently waiting to reinitialize it + return Err(EvictionCancelled::LostToDownload); } - } + }; + + permit }; + // now accesses to inner.get_or_init wait on the semaphore or the `_permit` + self.access_stats.record_residence_event( LayerResidenceStatus::Evicted, LayerResidenceEventReason::ResidenceChange, @@ -1086,6 +1090,7 @@ impl std::fmt::Display for NeedsDownload { pub(crate) struct DownloadedLayer { owner: Weak, kind: tokio::sync::OnceCell>, + version: usize, } impl std::fmt::Debug for DownloadedLayer { @@ -1093,6 +1098,7 @@ impl std::fmt::Debug for DownloadedLayer { f.debug_struct("DownloadedLayer") // owner omitted because it is always "Weak" .field("kind", &self.kind) + .field("version", &self.version) .finish() } } @@ -1100,7 +1106,7 @@ impl std::fmt::Debug for DownloadedLayer { impl Drop for DownloadedLayer { fn drop(&mut self) { if let Some(owner) = self.owner.upgrade() { - owner.on_downloaded_layer_drop(); + owner.on_downloaded_layer_drop(self.version); } else { // no need to do anything, we are shutting down } @@ -1458,6 +1464,9 @@ enum EvictionCancelled { VersionCheckFailed, FileNotFound, RemoveFailed, + AlreadyReinitialized, + /// Not evicted because of a pending reinitialization + LostToDownload, } impl EvictionCancelled { @@ -1468,6 +1477,8 @@ impl EvictionCancelled { EvictionCancelled::VersionCheckFailed => "version_check_fail", EvictionCancelled::FileNotFound => "file_not_found", EvictionCancelled::RemoveFailed => "remove_failed", + EvictionCancelled::AlreadyReinitialized => "already_reinitialized", + EvictionCancelled::LostToDownload => "lost_to_download", } } } From f7067a38b7d8b8e7413325f15edf9735be416856 Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Tue, 31 Oct 2023 10:00:23 -0700 Subject: [PATCH 55/69] compute_ctl: Assume --vm-monitor-addr arg is always present (#5611) It has a default value, so this should be sound. Treating its presence as semantically significant was leading to spurious warnings. --- compute_tools/src/bin/compute_ctl.rs | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index 04ac077c7b..263f71452d 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -278,8 +278,9 @@ fn main() -> Result<()> { if #[cfg(target_os = "linux")] { use std::env; use tokio_util::sync::CancellationToken; - use tracing::warn; - let vm_monitor_addr = matches.get_one::("vm-monitor-addr"); + let vm_monitor_addr = matches + .get_one::("vm-monitor-addr") + .expect("--vm-monitor-addr should always be set because it has a default arg"); let file_cache_connstr = matches.get_one::("filecache-connstr"); let cgroup = matches.get_one::("cgroup"); let file_cache_on_disk = matches.get_flag("file-cache-on-disk"); @@ -288,22 +289,16 @@ fn main() -> Result<()> { // Note: it seems like you can make a runtime in an inner scope and // if you start a task in it it won't be dropped. However, make it // in the outermost scope just to be safe. - let rt = match (env::var_os("AUTOSCALING"), vm_monitor_addr) { - (None, None) => None, - (None, Some(_)) => { - warn!("--vm-monitor-addr option set but AUTOSCALING env var not present"); - None - } - (Some(_), None) => { - panic!("AUTOSCALING env var present but --vm-monitor-addr option not set") - } - (Some(_), Some(_)) => Some( + let rt = if env::var_os("AUTOSCALING").is_some() { + Some( tokio::runtime::Builder::new_multi_thread() .worker_threads(4) .enable_all() .build() - .expect("failed to create tokio runtime for monitor"), - ), + .expect("failed to create tokio runtime for monitor") + ) + } else { + None }; // This token is used internally by the monitor to clean up all threads @@ -314,7 +309,7 @@ fn main() -> Result<()> { Box::leak(Box::new(vm_monitor::Args { cgroup: cgroup.cloned(), pgconnstr: file_cache_connstr.cloned(), - addr: vm_monitor_addr.cloned().unwrap(), + addr: vm_monitor_addr.clone(), file_cache_on_disk, })), token.clone(), From 726c8e673008c607788b9914013b3245b4eabba2 Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Fri, 11 Aug 2023 12:26:38 -0500 Subject: [PATCH 56/69] Add docs for updating Postgres for new minor versions --- docs/updating-postgres.md | 108 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 docs/updating-postgres.md diff --git a/docs/updating-postgres.md b/docs/updating-postgres.md new file mode 100644 index 0000000000..1868bbf5f7 --- /dev/null +++ b/docs/updating-postgres.md @@ -0,0 +1,108 @@ +# Updating Postgres + +## Minor Versions + +When upgrading to a new minor version of Postgres, please follow these steps: + +_Example: 15.4 is the new minor version to upgrade to from 15.3._ + +1. Clone the Neon Postgres repository if you have not done so already. + + ```shell + git clone git@github.com:neondatabase/postgres.git + ``` + +1. Add the Postgres upstream remote. + + ```shell + git remote add upstream https://git.postgresql.org/git/postgresql.git + ``` + +1. Create a new branch based on the stable branch you are updating. + + ```shell + git checkout -b my-branch REL_15_STABLE_neon + ``` + +1. Tag the last commit on the stable branch you are updating. + + ```shell + git tag REL_15_3_neon + ``` + +1. Push the new tag to the Neon Postgres repository. + + ```shell + git push origin REL_15_3_neon + ``` + +1. Find the release tags you're looking for. They are of the form `REL_X_Y`. + +1. Rebase the branch you created on the tag and resolve any conflicts. + + ```shell + git fetch upstream REL_15_4 + git rebase REL_15_4 + ``` + +1. Run the Postgres test suite to make sure our commits have not affected +Postgres in a negative way. + + ```shell + make check + # OR + meson test -C builddir + ``` + +1. Push your branch to the Neon Postgres repository. + + ```shell + git push origin my-branch + ``` + +1. Clone the Neon repository if you have not done so already. + + ```shell + git clone git@github.com:neondatabase/neon.git + ``` + +1. Create a new branch. + +1. Change the `revisions.json` file to point at the HEAD of your Postgres +branch. + +1. Update the Git submodule. + + ```shell + git submodule set-branch --branch my-branch vendor/postgres-v15 + git submodule update --remote vendor/postgres-v15 + ``` + +1. Run the Neon test suite to make sure that Neon is still good to go on this +minor Postgres release. + + ```shell + ./scripts/poetry -k pg15 + ``` + +1. Commit your changes. + +1. Create a pull request, and wait for CI to go green. + +1. Force push the rebased Postgres branches into the Neon Postgres repository. + + ```shell + git push --force origin my-branch:REL_15_STABLE_neon + ``` + + It may require disabling various branch protections. + +1. Update your Neon PR to point at the branches. + + ```shell + git submodule set-branch --branch REL_15_STABLE_neon vendor/postgres-v15 + git commit --amend --no-edit + git push --force origin + ``` + +1. Merge the pull request after getting approval(s) and CI completion. From 5952f350cb0c627a70998da3e9f90158d22fe8bc Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 31 Oct 2023 20:57:03 +0200 Subject: [PATCH 57/69] Always handle POLLHUP in walredo error poll loop (#5716) ## Problem test_stderr hangs on MacOS. See https://neondb.slack.com/archives/C036U0GRMRB/p1698438997903919 ## Summary of changes Always handle POLLHUP to prevent infinite loop. ## Checklist before requesting a review - [ ] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. ## Checklist before merging - [ ] Do not forget to reformat commit message to not include the above checklist Co-authored-by: Konstantin Knizhnik --- pageserver/src/walredo.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index cffd912e16..e1d699fcba 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -857,7 +857,8 @@ impl WalRedoProcess { let in_revents = stdin_pollfds[0].revents().unwrap(); if in_revents & (PollFlags::POLLERR | PollFlags::POLLOUT) != PollFlags::empty() { nwrite += proc.stdin.write(&writebuf[nwrite..])?; - } else if in_revents.contains(PollFlags::POLLHUP) { + } + if in_revents.contains(PollFlags::POLLHUP) { // We still have more data to write, but the process closed the pipe. anyhow::bail!("WAL redo process closed its stdin unexpectedly"); } @@ -907,7 +908,8 @@ impl WalRedoProcess { let out_revents = stdout_pollfds[0].revents().unwrap(); if out_revents & (PollFlags::POLLERR | PollFlags::POLLIN) != PollFlags::empty() { nresult += output.stdout.read(&mut resultbuf[nresult..])?; - } else if out_revents.contains(PollFlags::POLLHUP) { + } + if out_revents.contains(PollFlags::POLLHUP) { anyhow::bail!("WAL redo process closed its stdout unexpectedly"); } } From 4f0a8e92ad83cf0fad4436f3265d3eec2c601c96 Mon Sep 17 00:00:00 2001 From: Muhammet Yazici <44066377+mtyazici@users.noreply.github.com> Date: Wed, 1 Nov 2023 09:41:48 +0300 Subject: [PATCH 58/69] fix: Add bearer prefix to Authorization header (#5740) ## Problem Some requests with `Authorization` header did not properly set the `Bearer ` prefix. Problem explained here https://github.com/neondatabase/cloud/issues/6390. ## Summary of changes Added `Bearer ` prefix to missing requests. --- compute_tools/src/spec.rs | 2 +- pageserver/src/control_plane_client.rs | 5 ++++- proxy/src/console/provider/neon.rs | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index 6e4d4ccf49..3fa3cf055f 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -24,7 +24,7 @@ fn do_control_plane_request( ) -> Result { let resp = reqwest::blocking::Client::new() .get(uri) - .header("Authorization", jwt) + .header("Authorization", format!("Bearer {}", jwt)) .send() .map_err(|e| { ( diff --git a/pageserver/src/control_plane_client.rs b/pageserver/src/control_plane_client.rs index be4fcab75e..f50c19a629 100644 --- a/pageserver/src/control_plane_client.rs +++ b/pageserver/src/control_plane_client.rs @@ -57,7 +57,10 @@ impl ControlPlaneClient { if let Some(jwt) = &conf.control_plane_api_token { let mut headers = hyper::HeaderMap::new(); - headers.insert("Authorization", jwt.get_contents().parse().unwrap()); + headers.insert( + "Authorization", + format!("Bearer {}", jwt.get_contents()).parse().unwrap(), + ); client = client.default_headers(headers); } diff --git a/proxy/src/console/provider/neon.rs b/proxy/src/console/provider/neon.rs index 163cdfffc0..927fea0a13 100644 --- a/proxy/src/console/provider/neon.rs +++ b/proxy/src/console/provider/neon.rs @@ -49,7 +49,7 @@ impl Api { .endpoint .get("proxy_get_role_secret") .header("X-Request-ID", &request_id) - .header("Authorization", &self.jwt) + .header("Authorization", format!("Bearer {}", &self.jwt)) .query(&[("session_id", extra.session_id)]) .query(&[ ("application_name", extra.application_name), @@ -94,7 +94,7 @@ impl Api { .endpoint .get("proxy_wake_compute") .header("X-Request-ID", &request_id) - .header("Authorization", &self.jwt) + .header("Authorization", format!("Bearer {}", &self.jwt)) .query(&[("session_id", extra.session_id)]) .query(&[ ("application_name", extra.application_name), From e82d1ad6b8f00544d20614788f0f09fb02fb096d Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Wed, 1 Nov 2023 17:38:32 +0200 Subject: [PATCH 59/69] fix(layer): reinit on access before eviction happens (#5743) Right before merging, I added a loop to `fn LayerInner::get_or_maybe_download`, which was always supposed to be there. However I had forgotten to restart initialization instead of waiting for the eviction to happen to support original design goal of "eviction should always lose to redownload (or init)". This was wrong. After this fix, if `spawn_blocking` queue is blocked on something, nothing bad will happen. Part of #5737. --- pageserver/src/tenant/storage_layer/layer.rs | 130 +++++++++++-------- 1 file changed, 75 insertions(+), 55 deletions(-) diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index aaba9bd933..761fe311c6 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -337,31 +337,41 @@ enum ResidentOrWantedEvicted { } impl ResidentOrWantedEvicted { - fn get(&self) -> Option> { + /// If `Some` is returned, the ResidentOrWantedEvicted has been upgraded back from + /// `ResidentOrWantedEvicted::WantedEvicted` to `ResidentOrWantedEvicted::Resident`. + fn get_and_upgrade(&mut self) -> Option> { match self { ResidentOrWantedEvicted::Resident(strong) => Some(strong.clone()), ResidentOrWantedEvicted::WantedEvicted(weak, _) => match weak.upgrade() { Some(strong) => { LAYER_IMPL_METRICS.inc_raced_wanted_evicted_accesses(); + + *self = ResidentOrWantedEvicted::Resident(strong.clone()); + Some(strong) } None => None, }, } } + /// When eviction is first requested, drop down to holding a [`Weak`]. /// - /// Returns `true` if this was the first time eviction was requested. - fn downgrade(&mut self) -> bool { + /// Returns `Some` if this was the first time eviction was requested. Care should be taken to + /// drop the possibly last strong reference outside of the mutex of + /// heavier_once_cell::OnceCell. + fn downgrade(&mut self) -> Option> { match self { ResidentOrWantedEvicted::Resident(strong) => { let weak = Arc::downgrade(strong); - *self = ResidentOrWantedEvicted::WantedEvicted(weak, strong.version); - // returning the weak is not useful, because the drop could had already ran with - // the replacement above, and that will take care of cleaning the Option we are in - true + let mut temp = ResidentOrWantedEvicted::WantedEvicted(weak, strong.version); + std::mem::swap(self, &mut temp); + match temp { + ResidentOrWantedEvicted::Resident(strong) => Some(strong), + ResidentOrWantedEvicted::WantedEvicted(..) => unreachable!("just swapped"), + } } - ResidentOrWantedEvicted::WantedEvicted(..) => false, + ResidentOrWantedEvicted::WantedEvicted(..) => None, } } } @@ -563,20 +573,22 @@ impl LayerInner { let mut rx = self.status.subscribe(); - let res = - self.wanted_evicted - .compare_exchange(false, true, Ordering::Release, Ordering::Relaxed); + let strong = { + match self.inner.get() { + Some(mut either) => { + self.wanted_evicted.store(true, Ordering::Relaxed); + either.downgrade() + } + None => return Err(EvictionError::NotFound), + } + }; - if res.is_ok() { + if strong.is_some() { + // drop the DownloadedLayer outside of the holding the guard + drop(strong); LAYER_IMPL_METRICS.inc_started_evictions(); } - if self.get().is_none() { - // it was not evictable in the first place - // our store to the wanted_evicted does not matter; it will be reset by next download - return Err(EvictionError::NotFound); - } - match rx.recv().await { Ok(Status::Evicted) => Ok(()), Ok(Status::Downloaded) => Err(EvictionError::Downloaded), @@ -590,7 +602,8 @@ impl LayerInner { // // use however late (compared to the initial expressing of wanted) as the // "outcome" now - match self.get() { + LAYER_IMPL_METRICS.inc_broadcast_lagged(); + match self.inner.get() { Some(_) => Err(EvictionError::Downloaded), None => Ok(()), } @@ -605,6 +618,8 @@ impl LayerInner { allow_download: bool, ctx: Option<&RequestContext>, ) -> Result, DownloadError> { + let mut permit = None; + loop { let download = move || async move { // disable any scheduled but not yet running eviction deletions for this @@ -622,6 +637,8 @@ impl LayerInner { // check if we really need to be downloaded; could have been already downloaded by a // cancelled previous attempt. + // + // FIXME: what if it's a directory? that is currently needs_download == true let needs_download = self .needs_download() .await @@ -670,16 +687,37 @@ impl LayerInner { Ok(ResidentOrWantedEvicted::Resident(res)) }; - let locked = self.inner.get_or_init(download).await?; + let (weak, _permit) = { + // should we be able to give the permit to the `get_or_init`? would make sense. + drop(permit.take()); + let mut locked = self.inner.get_or_init(download).await?; - if let Some(strong) = Self::get_or_apply_evictedness(Some(locked), &self.wanted_evicted) - { - return Ok(strong); - } + if let Some(strong) = locked.get_and_upgrade() { + self.wanted_evicted.store(false, Ordering::Relaxed); + + // error out any `evict_and_wait` + drop(self.status.send(Status::Downloaded)); + + return Ok(strong); + } else { + // path to here: the evict_blocking is stuck on spawn_blocking queue. + // + // reset the contents, deactivating the eviction and causing a + // EvictionCancelled::LostToDownload or EvictionCancelled::VersionCheckFailed. + locked.take_and_deinit() + } + }; + + // unlock first, then drop the weak, but because upgrade failed, we + // know it cannot be a problem. + + assert!( + matches!(weak, ResidentOrWantedEvicted::WantedEvicted(..)), + "unexpected {weak:?}, ResidentOrWantedEvicted::get_and_upgrade has a bug" + ); + + permit = Some(_permit); - // the situation in which we might need to retry is that our init was ready - // immediatedly, but the DownloadedLayer had been dropped BUT failed to complete - // Self::evict_blocking LAYER_IMPL_METRICS.inc_retried_get_or_maybe_download(); } } @@ -812,33 +850,6 @@ impl LayerInner { } } - /// Access the current state without waiting for the file to be downloaded. - /// - /// Requires that we've initialized to state which is respective to the - /// actual residency state. - fn get(&self) -> Option> { - let locked = self.inner.get(); - Self::get_or_apply_evictedness(locked, &self.wanted_evicted) - } - - fn get_or_apply_evictedness( - guard: Option>, - wanted_evicted: &AtomicBool, - ) -> Option> { - if let Some(mut x) = guard { - if let Some(won) = x.get() { - // there are no guarantees that we will always get to observe a concurrent call - // to evict - if wanted_evicted.load(Ordering::Acquire) { - x.downgrade(); - } - return Some(won); - } - } - - None - } - async fn needs_download(&self) -> Result, std::io::Error> { match tokio::fs::metadata(&self.path).await { Ok(m) => Ok(self.is_file_present_and_good_size(&m).err()), @@ -872,7 +883,9 @@ impl LayerInner { fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { let layer_file_name = self.desc.filename().file_name(); - let remote = self.get().is_none(); + // this is not accurate: we could have the file locally but there was a cancellation + // and now we are not in sync, or we are currently downloading it. + let remote = self.inner.get().is_none(); let access_stats = self.access_stats.as_api_model(reset); @@ -1456,6 +1469,13 @@ impl LayerImplMetrics { .unwrap() .inc(); } + + fn inc_broadcast_lagged(&self) { + self.rare_counters + .get_metric_with_label_values(&["broadcast_lagged"]) + .unwrap() + .inc(); + } } enum EvictionCancelled { From 0b790b6d0058dcc47e2e56160a7f2c3b595483d9 Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Wed, 1 Nov 2023 17:02:58 -0400 Subject: [PATCH 60/69] Record wal size in import benchmark (#5755) --- test_runner/performance/test_bulk_insert.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test_runner/performance/test_bulk_insert.py b/test_runner/performance/test_bulk_insert.py index 62301f3919..46acec0f63 100644 --- a/test_runner/performance/test_bulk_insert.py +++ b/test_runner/performance/test_bulk_insert.py @@ -1,8 +1,10 @@ from contextlib import closing +from fixtures.benchmark_fixture import MetricReport from fixtures.compare_fixtures import NeonCompare, PgCompare from fixtures.pageserver.utils import wait_tenant_status_404 from fixtures.pg_version import PgVersion +from fixtures.types import Lsn # @@ -18,6 +20,8 @@ from fixtures.pg_version import PgVersion def test_bulk_insert(neon_with_baseline: PgCompare): env = neon_with_baseline + start_lsn = Lsn(env.pg.safe_psql("SELECT pg_current_wal_lsn()")[0][0]) + with closing(env.pg.connect()) as conn: with conn.cursor() as cur: cur.execute("create table huge (i int, j int);") @@ -31,6 +35,13 @@ def test_bulk_insert(neon_with_baseline: PgCompare): env.report_peak_memory_use() env.report_size() + # Report amount of wal written. Useful for comparing vanilla wal format vs + # neon wal format, measuring neon write amplification, etc. + end_lsn = Lsn(env.pg.safe_psql("SELECT pg_current_wal_lsn()")[0][0]) + wal_written_bytes = end_lsn - start_lsn + wal_written_mb = round(wal_written_bytes / (1024 * 1024)) + env.zenbenchmark.record("wal_written", wal_written_mb, "MB", MetricReport.TEST_PARAM) + # When testing neon, also check how long it takes the pageserver to reingest the # wal from safekeepers. If this number is close to total runtime, then the pageserver # is the bottleneck. From 2dca4c03fc8d12f83398408504eee7c404870d95 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 2 Nov 2023 10:06:32 +0200 Subject: [PATCH 61/69] feat(layer): cancellable get_or_maybe_download (#5744) With the layer implementation as was done in #4938, it is possible via cancellation to cause two concurrent downloads on the same path, due to how `RemoteTimelineClient::download_remote_layer` does tempfiles. Thread the init semaphore through the spawned task of downloading to make this impossible to happen. --- libs/utils/src/sync/heavier_once_cell.rs | 104 +++++++++++++------ pageserver/src/tenant/storage_layer/layer.rs | 48 ++++++--- 2 files changed, 105 insertions(+), 47 deletions(-) diff --git a/libs/utils/src/sync/heavier_once_cell.rs b/libs/utils/src/sync/heavier_once_cell.rs index 4d66a54c98..8a5aaf2ceb 100644 --- a/libs/utils/src/sync/heavier_once_cell.rs +++ b/libs/utils/src/sync/heavier_once_cell.rs @@ -60,8 +60,8 @@ impl OnceCell { /// Initialization is panic-safe and cancellation-safe. pub async fn get_or_init(&self, factory: F) -> Result, E> where - F: FnOnce() -> Fut, - Fut: std::future::Future>, + F: FnOnce(InitPermit) -> Fut, + Fut: std::future::Future>, { let sem = { let guard = self.inner.lock().unwrap(); @@ -72,28 +72,55 @@ impl OnceCell { }; let permit = sem.acquire_owned().await; - if permit.is_err() { - let guard = self.inner.lock().unwrap(); - assert!( - guard.value.is_some(), - "semaphore got closed, must be initialized" - ); - return Ok(Guard(guard)); - } else { - // now we try - let value = factory().await?; - let mut guard = self.inner.lock().unwrap(); - assert!( - guard.value.is_none(), - "we won permit, must not be initialized" - ); - guard.value = Some(value); - guard.init_semaphore.close(); - Ok(Guard(guard)) + match permit { + Ok(permit) => { + let permit = InitPermit(permit); + let (value, _permit) = factory(permit).await?; + + let guard = self.inner.lock().unwrap(); + + Ok(Self::set0(value, guard)) + } + Err(_closed) => { + let guard = self.inner.lock().unwrap(); + assert!( + guard.value.is_some(), + "semaphore got closed, must be initialized" + ); + return Ok(Guard(guard)); + } } } + /// Assuming a permit is held after previous call to [`Guard::take_and_deinit`], it can be used + /// to complete initializing the inner value. + /// + /// # Panics + /// + /// If the inner has already been initialized. + pub fn set(&self, value: T, _permit: InitPermit) -> Guard<'_, T> { + // cannot assert that this permit is for self.inner.semaphore + let guard = self.inner.lock().unwrap(); + + if guard.init_semaphore.try_acquire().is_ok() { + drop(guard); + panic!("semaphore is of wrong origin"); + } + + Self::set0(value, guard) + } + + fn set0(value: T, mut guard: std::sync::MutexGuard<'_, Inner>) -> Guard<'_, T> { + if guard.value.is_some() { + drop(guard); + unreachable!("we won permit, must not be initialized"); + } + guard.value = Some(value); + guard.init_semaphore.close(); + Guard(guard) + } + /// Returns a guard to an existing initialized value, if any. pub fn get(&self) -> Option> { let guard = self.inner.lock().unwrap(); @@ -135,7 +162,7 @@ impl<'a, T> Guard<'a, T> { /// /// The permit will be on a semaphore part of the new internal value, and any following /// [`OnceCell::get_or_init`] will wait on it to complete. - pub fn take_and_deinit(&mut self) -> (T, tokio::sync::OwnedSemaphorePermit) { + pub fn take_and_deinit(&mut self) -> (T, InitPermit) { let mut swapped = Inner::default(); let permit = swapped .init_semaphore @@ -145,11 +172,14 @@ impl<'a, T> Guard<'a, T> { std::mem::swap(&mut *self.0, &mut swapped); swapped .value - .map(|v| (v, permit)) + .map(|v| (v, InitPermit(permit))) .expect("guard is not created unless value has been initialized") } } +/// Type held by OnceCell (de)initializing task. +pub struct InitPermit(tokio::sync::OwnedSemaphorePermit); + #[cfg(test)] mod tests { use super::*; @@ -185,11 +215,11 @@ mod tests { barrier.wait().await; let won = { let g = cell - .get_or_init(|| { + .get_or_init(|permit| { counters.factory_got_to_run.fetch_add(1, Ordering::Relaxed); async { counters.future_polled.fetch_add(1, Ordering::Relaxed); - Ok::<_, Infallible>(i) + Ok::<_, Infallible>((i, permit)) } }) .await @@ -243,7 +273,7 @@ mod tests { deinitialization_started.wait().await; let started_at = tokio::time::Instant::now(); - cell.get_or_init(|| async { Ok::<_, Infallible>(reinit) }) + cell.get_or_init(|permit| async { Ok::<_, Infallible>((reinit, permit)) }) .await .unwrap(); @@ -258,18 +288,32 @@ mod tests { assert_eq!(*cell.get().unwrap(), reinit); } + #[test] + fn reinit_with_deinit_permit() { + let cell = Arc::new(OnceCell::new(42)); + + let (mol, permit) = cell.get().unwrap().take_and_deinit(); + cell.set(5, permit); + assert_eq!(*cell.get().unwrap(), 5); + + let (five, permit) = cell.get().unwrap().take_and_deinit(); + assert_eq!(5, five); + cell.set(mol, permit); + assert_eq!(*cell.get().unwrap(), 42); + } + #[tokio::test] async fn initialization_attemptable_until_ok() { let cell = OnceCell::default(); for _ in 0..10 { - cell.get_or_init(|| async { Err("whatever error") }) + cell.get_or_init(|_permit| async { Err("whatever error") }) .await .unwrap_err(); } let g = cell - .get_or_init(|| async { Ok::<_, Infallible>("finally success") }) + .get_or_init(|permit| async { Ok::<_, Infallible>(("finally success", permit)) }) .await .unwrap(); assert_eq!(*g, "finally success"); @@ -281,11 +325,11 @@ mod tests { let barrier = tokio::sync::Barrier::new(2); - let initializer = cell.get_or_init(|| async { + let initializer = cell.get_or_init(|permit| async { barrier.wait().await; futures::future::pending::<()>().await; - Ok::<_, Infallible>("never reached") + Ok::<_, Infallible>(("never reached", permit)) }); tokio::select! { @@ -298,7 +342,7 @@ mod tests { assert!(cell.get().is_none()); let g = cell - .get_or_init(|| async { Ok::<_, Infallible>("now initialized") }) + .get_or_init(|permit| async { Ok::<_, Infallible>(("now initialized", permit)) }) .await .unwrap(); assert_eq!(*g, "now initialized"); diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index 761fe311c6..94edfa6fe0 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -618,10 +618,10 @@ impl LayerInner { allow_download: bool, ctx: Option<&RequestContext>, ) -> Result, DownloadError> { - let mut permit = None; + let mut init_permit = None; loop { - let download = move || async move { + let download = move |permit| async move { // disable any scheduled but not yet running eviction deletions for this let next_version = 1 + self.version.fetch_add(1, Ordering::Relaxed); @@ -644,7 +644,7 @@ impl LayerInner { .await .map_err(DownloadError::PreStatFailed)?; - if let Some(reason) = needs_download { + let permit = if let Some(reason) = needs_download { // only reset this after we've decided we really need to download. otherwise it'd // be impossible to mark cancelled downloads for eviction, like one could imagine // we would like to do for prefetching which was not needed. @@ -666,12 +666,14 @@ impl LayerInner { return Err(DownloadError::DownloadRequired); } - self.spawn_download_and_wait(timeline).await?; + self.spawn_download_and_wait(timeline, permit).await? } else { // the file is present locally, probably by a previous but cancelled call to // get_or_maybe_download. alternatively we might be running without remote storage. LAYER_IMPL_METRICS.inc_init_needed_no_download(); - } + + permit + }; let res = Arc::new(DownloadedLayer { owner: Arc::downgrade(self), @@ -684,12 +686,21 @@ impl LayerInner { LayerResidenceEventReason::ResidenceChange, ); - Ok(ResidentOrWantedEvicted::Resident(res)) + Ok((ResidentOrWantedEvicted::Resident(res), permit)) }; - let (weak, _permit) = { - // should we be able to give the permit to the `get_or_init`? would make sense. - drop(permit.take()); + if let Some(init_permit) = init_permit.take() { + // use the already held initialization permit because it is impossible to hit the + // below paths anymore essentially limiting the max loop iterations to 2. + let (value, init_permit) = download(init_permit).await?; + let mut guard = self.inner.set(value, init_permit); + let strong = guard + .get_and_upgrade() + .expect("init creates strong reference, we held the init permit"); + return Ok(strong); + } + + let (weak, permit) = { let mut locked = self.inner.get_or_init(download).await?; if let Some(strong) = locked.get_and_upgrade() { @@ -716,7 +727,7 @@ impl LayerInner { "unexpected {weak:?}, ResidentOrWantedEvicted::get_and_upgrade has a bug" ); - permit = Some(_permit); + init_permit = Some(permit); LAYER_IMPL_METRICS.inc_retried_get_or_maybe_download(); } @@ -752,10 +763,12 @@ impl LayerInner { async fn spawn_download_and_wait( self: &Arc, timeline: Arc, - ) -> Result<(), DownloadError> { + permit: heavier_once_cell::InitPermit, + ) -> Result { let task_name = format!("download layer {}", self); let (tx, rx) = tokio::sync::oneshot::channel(); + // this is sadly needed because of task_mgr::shutdown_tasks, otherwise we cannot // block tenant::mgr::remove_tenant_from_memory. @@ -789,9 +802,9 @@ impl LayerInner { } }; - if let Err(res) = tx.send(result) { + if let Err(res) = tx.send((result, permit)) { match res { - Ok(()) => { + (Ok(()), _) => { // our caller is cancellation safe so this is fine; if someone // else requests the layer, they'll find it already downloaded // or redownload. @@ -802,7 +815,7 @@ impl LayerInner { tracing::info!("layer file download completed after requester had cancelled"); LAYER_IMPL_METRICS.inc_download_completed_without_requester(); }, - Err(e) => { + (Err(e), _) => { // our caller is cancellation safe, but we might be racing with // another attempt to initialize. before we have cancellation // token support: these attempts should converge regardless of @@ -818,7 +831,7 @@ impl LayerInner { .in_current_span(), ); match rx.await { - Ok(Ok(())) => { + Ok((Ok(()), permit)) => { if let Some(reason) = self .needs_download() .await @@ -830,9 +843,10 @@ impl LayerInner { self.consecutive_failures.store(0, Ordering::Relaxed); - Ok(()) + Ok(permit) } - Ok(Err(e)) => { + Ok((Err(e), _permit)) => { + // FIXME: this should be with the spawned task and be cancellation sensitive let consecutive_failures = self.consecutive_failures.fetch_add(1, Ordering::Relaxed); tracing::error!(consecutive_failures, "layer file download failed: {e:#}"); From 5650138532e5e3ca4a06f15f3ed79d6a3545f5a0 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 2 Nov 2023 09:14:26 +0000 Subject: [PATCH 62/69] pageserver: helpers for explicitly dying on fatal I/O errors (#5651) Following from discussion on https://github.com/neondatabase/neon/pull/5436 where hacking an implicit die-on-fatal-io behavior into an Error type was a source of disagreement -- in this PR, dying on fatal I/O errors is explicit, with `fatal_err` and `maybe_fatal_err` helpers in the `MaybeFatalIo` trait, which is implemented for std::io::Result. To enable this approach with `crashsafe_overwrite`, the return type of that function is changed to std::io::Result -- the previous error enum for this function was not used for any logic, and the utility of saying exactly which step in the function failed is outweighed by the hygiene of having an I/O funciton return an io::Result. The initial use case for these helpers is the deletion queue. --- pageserver/src/deletion_queue.rs | 6 +- pageserver/src/deletion_queue/list_writer.rs | 31 ++--- pageserver/src/deletion_queue/validator.rs | 14 +- pageserver/src/virtual_file.rs | 135 +++++++++++-------- 4 files changed, 104 insertions(+), 82 deletions(-) diff --git a/pageserver/src/deletion_queue.rs b/pageserver/src/deletion_queue.rs index 22efa23f10..7b2db929fa 100644 --- a/pageserver/src/deletion_queue.rs +++ b/pageserver/src/deletion_queue.rs @@ -10,6 +10,7 @@ use crate::control_plane_client::ControlPlaneGenerationsApi; use crate::metrics; use crate::tenant::remote_timeline_client::remote_layer_path; use crate::tenant::remote_timeline_client::remote_timeline_path; +use crate::virtual_file::MaybeFatalIo; use crate::virtual_file::VirtualFile; use anyhow::Context; use camino::Utf8PathBuf; @@ -271,7 +272,9 @@ impl DeletionHeader { let temp_path = path_with_suffix_extension(&header_path, TEMP_SUFFIX); VirtualFile::crashsafe_overwrite(&header_path, &temp_path, &header_bytes) .await - .map_err(Into::into) + .maybe_fatal_err("save deletion header")?; + + Ok(()) } } @@ -360,6 +363,7 @@ impl DeletionList { let bytes = serde_json::to_vec(self).expect("Failed to serialize deletion list"); VirtualFile::crashsafe_overwrite(&path, &temp_path, &bytes) .await + .maybe_fatal_err("save deletion list") .map_err(Into::into) } } diff --git a/pageserver/src/deletion_queue/list_writer.rs b/pageserver/src/deletion_queue/list_writer.rs index 6727957b2a..28daae2da5 100644 --- a/pageserver/src/deletion_queue/list_writer.rs +++ b/pageserver/src/deletion_queue/list_writer.rs @@ -34,6 +34,8 @@ use crate::deletion_queue::TEMP_SUFFIX; use crate::metrics; use crate::tenant::remote_timeline_client::remote_layer_path; use crate::tenant::storage_layer::LayerFileName; +use crate::virtual_file::on_fatal_io_error; +use crate::virtual_file::MaybeFatalIo; // The number of keys in a DeletionList before we will proactively persist it // (without reaching a flush deadline). This aims to deliver objects of the order @@ -195,7 +197,7 @@ impl ListWriter { debug!("Deletion header {header_path} not found, first start?"); Ok(None) } else { - Err(anyhow::anyhow!(e)) + on_fatal_io_error(&e, "reading deletion header"); } } } @@ -216,16 +218,9 @@ impl ListWriter { self.pending.sequence = validated_sequence + 1; let deletion_directory = self.conf.deletion_prefix(); - let mut dir = match tokio::fs::read_dir(&deletion_directory).await { - Ok(d) => d, - Err(e) => { - warn!("Failed to open deletion list directory {deletion_directory}: {e:#}"); - - // Give up: if we can't read the deletion list directory, we probably can't - // write lists into it later, so the queue won't work. - return Err(e.into()); - } - }; + let mut dir = tokio::fs::read_dir(&deletion_directory) + .await + .fatal_err("read deletion directory"); let list_name_pattern = Regex::new("(?[a-zA-Z0-9]{16})-(?[a-zA-Z0-9]{2}).list").unwrap(); @@ -233,7 +228,7 @@ impl ListWriter { let temp_extension = format!(".{TEMP_SUFFIX}"); let header_path = self.conf.deletion_header_path(); let mut seqs: Vec = Vec::new(); - while let Some(dentry) = dir.next_entry().await? { + while let Some(dentry) = dir.next_entry().await.fatal_err("read deletion dentry") { let file_name = dentry.file_name(); let dentry_str = file_name.to_string_lossy(); @@ -246,11 +241,9 @@ impl ListWriter { info!("Cleaning up temporary file {dentry_str}"); let absolute_path = deletion_directory.join(dentry.file_name().to_str().expect("non-Unicode path")); - if let Err(e) = tokio::fs::remove_file(&absolute_path).await { - // Non-fatal error: we will just leave the file behind but not - // try and load it. - warn!("Failed to clean up temporary file {absolute_path}: {e:#}"); - } + tokio::fs::remove_file(&absolute_path) + .await + .fatal_err("delete temp file"); continue; } @@ -290,7 +283,9 @@ impl ListWriter { for s in seqs { let list_path = self.conf.deletion_list_path(s); - let list_bytes = tokio::fs::read(&list_path).await?; + let list_bytes = tokio::fs::read(&list_path) + .await + .fatal_err("read deletion list"); let mut deletion_list = match serde_json::from_slice::(&list_bytes) { Ok(l) => l, diff --git a/pageserver/src/deletion_queue/validator.rs b/pageserver/src/deletion_queue/validator.rs index a2cbfb9dc7..72bdbdefd6 100644 --- a/pageserver/src/deletion_queue/validator.rs +++ b/pageserver/src/deletion_queue/validator.rs @@ -28,6 +28,7 @@ use crate::config::PageServerConf; use crate::control_plane_client::ControlPlaneGenerationsApi; use crate::control_plane_client::RetryForeverError; use crate::metrics; +use crate::virtual_file::MaybeFatalIo; use super::deleter::DeleterMessage; use super::DeletionHeader; @@ -287,16 +288,9 @@ where async fn cleanup_lists(&mut self, list_paths: Vec) { for list_path in list_paths { debug!("Removing deletion list {list_path}"); - - if let Err(e) = tokio::fs::remove_file(&list_path).await { - // Unexpected: we should have permissions and nothing else should - // be touching these files. We will leave the file behind. Subsequent - // pageservers will try and load it again: hopefully whatever storage - // issue (probably permissions) has been fixed by then. - tracing::error!("Failed to delete {list_path}: {e:#}"); - metrics::DELETION_QUEUE.unexpected_errors.inc(); - break; - } + tokio::fs::remove_file(&list_path) + .await + .fatal_err("remove deletion list"); } } diff --git a/pageserver/src/virtual_file.rs b/pageserver/src/virtual_file.rs index a2e8f30e15..b58b883ab6 100644 --- a/pageserver/src/virtual_file.rs +++ b/pageserver/src/virtual_file.rs @@ -19,6 +19,7 @@ use std::io::{Error, ErrorKind, Seek, SeekFrom}; use std::os::unix::fs::FileExt; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{RwLock, RwLockWriteGuard}; +use utils::fs_ext; /// /// A virtual file descriptor. You can use this just like std::fs::File, but internally @@ -173,37 +174,78 @@ impl OpenFiles { } } -#[derive(Debug, thiserror::Error)] -pub enum CrashsafeOverwriteError { - #[error("final path has no parent dir")] - FinalPathHasNoParentDir, - #[error("remove tempfile")] - RemovePreviousTempfile(#[source] std::io::Error), - #[error("create tempfile")] - CreateTempfile(#[source] std::io::Error), - #[error("write tempfile")] - WriteContents(#[source] std::io::Error), - #[error("sync tempfile")] - SyncTempfile(#[source] std::io::Error), - #[error("rename tempfile to final path")] - RenameTempfileToFinalPath(#[source] std::io::Error), - #[error("open final path parent dir")] - OpenFinalPathParentDir(#[source] std::io::Error), - #[error("sync final path parent dir")] - SyncFinalPathParentDir(#[source] std::io::Error), +/// Identify error types that should alwways terminate the process. Other +/// error types may be elegible for retry. +pub(crate) fn is_fatal_io_error(e: &std::io::Error) -> bool { + use nix::errno::Errno::*; + match e.raw_os_error().map(nix::errno::from_i32) { + Some(EIO) => { + // Terminate on EIO because we no longer trust the device to store + // data safely, or to uphold persistence guarantees on fsync. + true + } + Some(EROFS) => { + // Terminate on EROFS because a filesystem is usually remounted + // readonly when it has experienced some critical issue, so the same + // logic as EIO applies. + true + } + Some(EACCES) => { + // Terminate on EACCESS because we should always have permissions + // for our own data dir: if we don't, then we can't do our job and + // need administrative intervention to fix permissions. Terminating + // is the best way to make sure we stop cleanly rather than going + // into infinite retry loops, and will make it clear to the outside + // world that we need help. + true + } + _ => { + // Treat all other local file I/O errors are retryable. This includes: + // - ENOSPC: we stay up and wait for eviction to free some space + // - EINVAL, EBADF, EBADFD: this is a code bug, not a filesystem/hardware issue + // - WriteZero, Interrupted: these are used internally VirtualFile + false + } + } } -impl CrashsafeOverwriteError { - /// Returns true iff the new contents are durably stored. - pub fn are_new_contents_durable(&self) -> bool { + +/// Call this when the local filesystem gives us an error with an external +/// cause: this includes EIO, EROFS, and EACCESS: all these indicate either +/// bad storage or bad configuration, and we can't fix that from inside +/// a running process. +pub(crate) fn on_fatal_io_error(e: &std::io::Error, context: &str) -> ! { + tracing::error!("Fatal I/O error: {e}: {context})"); + std::process::abort(); +} + +pub(crate) trait MaybeFatalIo { + fn maybe_fatal_err(self, context: &str) -> std::io::Result; + fn fatal_err(self, context: &str) -> T; +} + +impl MaybeFatalIo for std::io::Result { + /// Terminate the process if the result is an error of a fatal type, else pass it through + /// + /// This is appropriate for writes, where we typically want to die on EIO/ACCES etc, but + /// not on ENOSPC. + fn maybe_fatal_err(self, context: &str) -> std::io::Result { + if let Err(e) = &self { + if is_fatal_io_error(e) { + on_fatal_io_error(e, context); + } + } + self + } + + /// Terminate the process on any I/O error. + /// + /// This is appropriate for reads on files that we know exist: they should always work. + fn fatal_err(self, context: &str) -> T { match self { - Self::FinalPathHasNoParentDir => false, - Self::RemovePreviousTempfile(_) => false, - Self::CreateTempfile(_) => false, - Self::WriteContents(_) => false, - Self::SyncTempfile(_) => false, - Self::RenameTempfileToFinalPath(_) => false, - Self::OpenFinalPathParentDir(_) => false, - Self::SyncFinalPathParentDir(_) => true, + Ok(v) => v, + Err(e) => { + on_fatal_io_error(&e, context); + } } } } @@ -284,15 +326,13 @@ impl VirtualFile { final_path: &Utf8Path, tmp_path: &Utf8Path, content: &[u8], - ) -> Result<(), CrashsafeOverwriteError> { + ) -> std::io::Result<()> { let Some(final_path_parent) = final_path.parent() else { - return Err(CrashsafeOverwriteError::FinalPathHasNoParentDir); + return Err(std::io::Error::from_raw_os_error( + nix::errno::Errno::EINVAL as i32, + )); }; - match std::fs::remove_file(tmp_path) { - Ok(()) => {} - Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} - Err(e) => return Err(CrashsafeOverwriteError::RemovePreviousTempfile(e)), - } + std::fs::remove_file(tmp_path).or_else(fs_ext::ignore_not_found)?; let mut file = Self::open_with_options( tmp_path, OpenOptions::new() @@ -301,31 +341,20 @@ impl VirtualFile { // we bail out instead of causing damage. .create_new(true), ) - .await - .map_err(CrashsafeOverwriteError::CreateTempfile)?; - file.write_all(content) - .await - .map_err(CrashsafeOverwriteError::WriteContents)?; - file.sync_all() - .await - .map_err(CrashsafeOverwriteError::SyncTempfile)?; + .await?; + file.write_all(content).await?; + file.sync_all().await?; drop(file); // before the rename, that's important! // renames are atomic - std::fs::rename(tmp_path, final_path) - .map_err(CrashsafeOverwriteError::RenameTempfileToFinalPath)?; + std::fs::rename(tmp_path, final_path)?; // Only open final path parent dirfd now, so that this operation only // ever holds one VirtualFile fd at a time. That's important because // the current `find_victim_slot` impl might pick the same slot for both // VirtualFile., and it eventually does a blocking write lock instead of // try_lock. let final_parent_dirfd = - Self::open_with_options(final_path_parent, OpenOptions::new().read(true)) - .await - .map_err(CrashsafeOverwriteError::OpenFinalPathParentDir)?; - final_parent_dirfd - .sync_all() - .await - .map_err(CrashsafeOverwriteError::SyncFinalPathParentDir)?; + Self::open_with_options(final_path_parent, OpenOptions::new().read(true)).await?; + final_parent_dirfd.sync_all().await?; Ok(()) } From 3737fe3a4bf50d8e24aa6eea400e6464aa73bcda Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 2 Nov 2023 13:03:38 +0200 Subject: [PATCH 63/69] fix(layer): error out early if layer path is non-file (#5756) In an earlier PR https://github.com/neondatabase/neon/pull/5743#discussion_r1378625244 I added a FIXME and there's a simple solution suggested by @jcsp, so implement it. Wondering why I did not implement this originally, there is no concept of a permanent failure, so this failure will happen quite often. I don't think the frequency is a problem however. Sadly for std::fs::FileType there is only decimal and hex formatting, no octal. --- pageserver/src/tenant/storage_layer/layer.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index 94edfa6fe0..fc4ba75dfc 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -637,14 +637,16 @@ impl LayerInner { // check if we really need to be downloaded; could have been already downloaded by a // cancelled previous attempt. - // - // FIXME: what if it's a directory? that is currently needs_download == true let needs_download = self .needs_download() .await .map_err(DownloadError::PreStatFailed)?; let permit = if let Some(reason) = needs_download { + if let NeedsDownload::NotFile(ft) = reason { + return Err(DownloadError::NotFile(ft)); + } + // only reset this after we've decided we really need to download. otherwise it'd // be impossible to mark cancelled downloads for eviction, like one could imagine // we would like to do for prefetching which was not needed. @@ -883,7 +885,7 @@ impl LayerInner { fn is_file_present_and_good_size(&self, m: &std::fs::Metadata) -> Result<(), NeedsDownload> { // in future, this should include sha2-256 validation of the file. if !m.is_file() { - Err(NeedsDownload::NotFile) + Err(NeedsDownload::NotFile(m.file_type())) } else if m.len() != self.desc.file_size { Err(NeedsDownload::WrongSize { actual: m.len(), @@ -1082,6 +1084,8 @@ enum DownloadError { ContextAndConfigReallyDeniesDownloads, #[error("downloading is really required but not allowed by this method")] DownloadRequired, + #[error("layer path exists, but it is not a file: {0:?}")] + NotFile(std::fs::FileType), /// Why no error here? Because it will be reported by page_service. We should had also done /// retries already. #[error("downloading evicted layer file failed")] @@ -1097,7 +1101,7 @@ enum DownloadError { #[derive(Debug, PartialEq)] pub(crate) enum NeedsDownload { NotFound, - NotFile, + NotFile(std::fs::FileType), WrongSize { actual: u64, expected: u64 }, } @@ -1105,7 +1109,7 @@ impl std::fmt::Display for NeedsDownload { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { NeedsDownload::NotFound => write!(f, "file was not found"), - NeedsDownload::NotFile => write!(f, "path is not a file"), + NeedsDownload::NotFile(ft) => write!(f, "path is not a file; {ft:?}"), NeedsDownload::WrongSize { actual, expected } => { write!(f, "file size mismatch {actual} vs. {expected}") } From 098d3111a5c4fb057571246d637edfa54d95b334 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 2 Nov 2023 15:06:14 +0200 Subject: [PATCH 64/69] fix(layer): get_and_upgrade and metrics (#5767) when introducing `get_and_upgrade` I forgot that an `evict_and_wait` would had already incremented the counter for started evictions, but an upgrade would just "silently" cancel the eviction as no drop would ever run. these metrics are likely sources for alerts with the next release, so it's important to keep them correct. --- pageserver/src/tenant/storage_layer/layer.rs | 27 ++++++++++++-------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index fc4ba75dfc..b320c02f9b 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -337,18 +337,16 @@ enum ResidentOrWantedEvicted { } impl ResidentOrWantedEvicted { - /// If `Some` is returned, the ResidentOrWantedEvicted has been upgraded back from - /// `ResidentOrWantedEvicted::WantedEvicted` to `ResidentOrWantedEvicted::Resident`. - fn get_and_upgrade(&mut self) -> Option> { + fn get_and_upgrade(&mut self) -> Option<(Arc, bool)> { match self { - ResidentOrWantedEvicted::Resident(strong) => Some(strong.clone()), + ResidentOrWantedEvicted::Resident(strong) => Some((strong.clone(), false)), ResidentOrWantedEvicted::WantedEvicted(weak, _) => match weak.upgrade() { Some(strong) => { LAYER_IMPL_METRICS.inc_raced_wanted_evicted_accesses(); *self = ResidentOrWantedEvicted::Resident(strong.clone()); - Some(strong) + Some((strong, true)) } None => None, }, @@ -696,7 +694,7 @@ impl LayerInner { // below paths anymore essentially limiting the max loop iterations to 2. let (value, init_permit) = download(init_permit).await?; let mut guard = self.inner.set(value, init_permit); - let strong = guard + let (strong, _upgraded) = guard .get_and_upgrade() .expect("init creates strong reference, we held the init permit"); return Ok(strong); @@ -705,11 +703,17 @@ impl LayerInner { let (weak, permit) = { let mut locked = self.inner.get_or_init(download).await?; - if let Some(strong) = locked.get_and_upgrade() { - self.wanted_evicted.store(false, Ordering::Relaxed); + if let Some((strong, upgraded)) = locked.get_and_upgrade() { + if upgraded { + // when upgraded back, the Arc is still available, but + // previously a `evict_and_wait` was received. + self.wanted_evicted.store(false, Ordering::Relaxed); - // error out any `evict_and_wait` - drop(self.status.send(Status::Downloaded)); + // error out any `evict_and_wait` + drop(self.status.send(Status::Downloaded)); + LAYER_IMPL_METRICS + .inc_eviction_cancelled(EvictionCancelled::UpgradedBackOnAccess); + } return Ok(strong); } else { @@ -1505,6 +1509,8 @@ enum EvictionCancelled { AlreadyReinitialized, /// Not evicted because of a pending reinitialization LostToDownload, + /// After eviction, there was a new layer access which cancelled the eviction. + UpgradedBackOnAccess, } impl EvictionCancelled { @@ -1517,6 +1523,7 @@ impl EvictionCancelled { EvictionCancelled::RemoveFailed => "remove_failed", EvictionCancelled::AlreadyReinitialized => "already_reinitialized", EvictionCancelled::LostToDownload => "lost_to_download", + EvictionCancelled::UpgradedBackOnAccess => "upgraded_back_on_access", } } } From 51570114ea5dc5c5c73edc5460b3cf2ce76e2970 Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Thu, 2 Nov 2023 10:43:59 -0400 Subject: [PATCH 65/69] Remove outdated and flaky perf test (#5762) --- scripts/benchmark_durations.py | 1 - test_runner/performance/test_startup.py | 49 ------------------------- 2 files changed, 50 deletions(-) diff --git a/scripts/benchmark_durations.py b/scripts/benchmark_durations.py index 781d0d2d1e..7f05d72a03 100755 --- a/scripts/benchmark_durations.py +++ b/scripts/benchmark_durations.py @@ -81,7 +81,6 @@ FALLBACK_DURATION = { "test_runner/performance/test_seqscans.py::test_seqscans[vanilla-100000-100-0]": 0.55, "test_runner/performance/test_seqscans.py::test_seqscans[vanilla-10000000-1-0]": 12.189, "test_runner/performance/test_seqscans.py::test_seqscans[vanilla-10000000-1-4]": 13.899, - "test_runner/performance/test_startup.py::test_startup": 890.114, "test_runner/performance/test_startup.py::test_startup_simple": 2.51, "test_runner/performance/test_wal_backpressure.py::test_heavy_write_workload[neon_off-10-5-5]": 527.245, "test_runner/performance/test_wal_backpressure.py::test_heavy_write_workload[neon_on-10-5-5]": 583.46, diff --git a/test_runner/performance/test_startup.py b/test_runner/performance/test_startup.py index 366b9b0e68..301078d984 100644 --- a/test_runner/performance/test_startup.py +++ b/test_runner/performance/test_startup.py @@ -1,6 +1,3 @@ -from contextlib import closing - -import pytest import requests from fixtures.benchmark_fixture import MetricReport, NeonBenchmarker from fixtures.neon_fixtures import NeonEnvBuilder @@ -81,49 +78,3 @@ def test_startup_simple(neon_env_builder: NeonEnvBuilder, zenbenchmark: NeonBenc # Imitate optimizations that console would do for the second start endpoint.respec(skip_pg_catalog_updates=True) - - -# This test sometimes runs for longer than the global 5 minute timeout. -@pytest.mark.timeout(900) -def test_startup(neon_env_builder: NeonEnvBuilder, zenbenchmark: NeonBenchmarker): - neon_env_builder.num_safekeepers = 3 - env = neon_env_builder.init_start() - - # Start - env.neon_cli.create_branch("test_startup") - with zenbenchmark.record_duration("startup_time"): - endpoint = env.endpoints.create_start("test_startup") - endpoint.safe_psql("select 1;") - - # Restart - endpoint.stop_and_destroy() - with zenbenchmark.record_duration("restart_time"): - endpoint.create_start("test_startup") - endpoint.safe_psql("select 1;") - - # Fill up - num_rows = 1000000 # 30 MB - num_tables = 100 - with closing(endpoint.connect()) as conn: - with conn.cursor() as cur: - for i in range(num_tables): - cur.execute(f"create table t_{i} (i integer);") - cur.execute(f"insert into t_{i} values (generate_series(1,{num_rows}));") - - # Read - with zenbenchmark.record_duration("read_time"): - endpoint.safe_psql("select * from t_0;") - - # Read again - with zenbenchmark.record_duration("second_read_time"): - endpoint.safe_psql("select * from t_0;") - - # Restart - endpoint.stop_and_destroy() - with zenbenchmark.record_duration("restart_with_data"): - endpoint.create_start("test_startup") - endpoint.safe_psql("select 1;") - - # Read - with zenbenchmark.record_duration("read_after_restart"): - endpoint.safe_psql("select * from t_0;") From 367971a0e9671abd2bcb45fbfa168ae6de585bb5 Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Thu, 2 Nov 2023 09:06:16 -0700 Subject: [PATCH 66/69] vm-monitor: Remove support for file cache in tmpfs (#5617) ref neondatabase/cloud#7516. We switched everything over to file cache on disk, now time to remove support for having it in tmpfs. --- compute_tools/src/bin/compute_ctl.rs | 4 ++-- libs/vm_monitor/src/filecache.rs | 24 ++++--------------- libs/vm_monitor/src/lib.rs | 10 -------- libs/vm_monitor/src/runner.rs | 35 +++++++--------------------- 4 files changed, 15 insertions(+), 58 deletions(-) diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index 263f71452d..81d4320b14 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -283,7 +283,6 @@ fn main() -> Result<()> { .expect("--vm-monitor-addr should always be set because it has a default arg"); let file_cache_connstr = matches.get_one::("filecache-connstr"); let cgroup = matches.get_one::("cgroup"); - let file_cache_on_disk = matches.get_flag("file-cache-on-disk"); // Only make a runtime if we need to. // Note: it seems like you can make a runtime in an inner scope and @@ -310,7 +309,6 @@ fn main() -> Result<()> { cgroup: cgroup.cloned(), pgconnstr: file_cache_connstr.cloned(), addr: vm_monitor_addr.clone(), - file_cache_on_disk, })), token.clone(), )) @@ -482,6 +480,8 @@ fn cli() -> clap::Command { .value_name("FILECACHE_CONNSTR"), ) .arg( + // DEPRECATED, NO LONGER DOES ANYTHING. + // See https://github.com/neondatabase/cloud/issues/7516 Arg::new("file-cache-on-disk") .long("file-cache-on-disk") .action(clap::ArgAction::SetTrue), diff --git a/libs/vm_monitor/src/filecache.rs b/libs/vm_monitor/src/filecache.rs index 3a860500f1..fe71e11197 100644 --- a/libs/vm_monitor/src/filecache.rs +++ b/libs/vm_monitor/src/filecache.rs @@ -21,11 +21,6 @@ pub struct FileCacheState { #[derive(Debug)] pub struct FileCacheConfig { - /// Whether the file cache is *actually* stored in memory (e.g. by writing to - /// a tmpfs or shmem file). If true, the size of the file cache will be counted against the - /// memory available for the cgroup. - pub(crate) in_memory: bool, - /// The size of the file cache, in terms of the size of the resource it consumes /// (currently: only memory) /// @@ -59,22 +54,9 @@ pub struct FileCacheConfig { spread_factor: f64, } -impl FileCacheConfig { - pub fn default_in_memory() -> Self { +impl Default for FileCacheConfig { + fn default() -> Self { Self { - in_memory: true, - // 75 % - resource_multiplier: 0.75, - // 640 MiB; (512 + 128) - min_remaining_after_cache: NonZeroU64::new(640 * MiB).unwrap(), - // ensure any increase in file cache size is split 90-10 with 10% to other memory - spread_factor: 0.1, - } - } - - pub fn default_on_disk() -> Self { - Self { - in_memory: false, resource_multiplier: 0.75, // 256 MiB - lower than when in memory because overcommitting is safe; if we don't have // memory, the kernel will just evict from its page cache, rather than e.g. killing @@ -83,7 +65,9 @@ impl FileCacheConfig { spread_factor: 0.1, } } +} +impl FileCacheConfig { /// Make sure fields of the config are consistent. pub fn validate(&self) -> anyhow::Result<()> { // Single field validity diff --git a/libs/vm_monitor/src/lib.rs b/libs/vm_monitor/src/lib.rs index 1cbfdc6ba6..a844f78bd6 100644 --- a/libs/vm_monitor/src/lib.rs +++ b/libs/vm_monitor/src/lib.rs @@ -39,16 +39,6 @@ pub struct Args { #[arg(short, long)] pub pgconnstr: Option, - /// Flag to signal that the Postgres file cache is on disk (i.e. not in memory aside from the - /// kernel's page cache), and therefore should not count against available memory. - // - // NB: Ideally this flag would directly refer to whether the file cache is in memory (rather - // than a roundabout way, via whether it's on disk), but in order to be backwards compatible - // during the switch away from an in-memory file cache, we had to default to the previous - // behavior. - #[arg(long)] - pub file_cache_on_disk: bool, - /// The address we should listen on for connection requests. For the /// agent, this is 0.0.0.0:10301. For the informant, this is 127.0.0.1:10369. #[arg(short, long)] diff --git a/libs/vm_monitor/src/runner.rs b/libs/vm_monitor/src/runner.rs index 99bea6d7ca..f162f53d24 100644 --- a/libs/vm_monitor/src/runner.rs +++ b/libs/vm_monitor/src/runner.rs @@ -156,10 +156,7 @@ impl Runner { // memory limits. if let Some(connstr) = &args.pgconnstr { info!("initializing file cache"); - let config = match args.file_cache_on_disk { - true => FileCacheConfig::default_on_disk(), - false => FileCacheConfig::default_in_memory(), - }; + let config = FileCacheConfig::default(); let mut file_cache = FileCacheState::new(connstr, config, token.clone()) .await @@ -187,10 +184,7 @@ impl Runner { info!("file cache size actually got set to {actual_size}") } - if args.file_cache_on_disk { - file_cache_disk_size = actual_size; - } - + file_cache_disk_size = actual_size; state.filecache = Some(file_cache); } @@ -239,17 +233,11 @@ impl Runner { let requested_mem = target.mem; let usable_system_memory = requested_mem.saturating_sub(self.config.sys_buffer_bytes); - let (expected_file_cache_size, expected_file_cache_disk_size) = self + let expected_file_cache_size = self .filecache .as_ref() - .map(|file_cache| { - let size = file_cache.config.calculate_cache_size(usable_system_memory); - match file_cache.config.in_memory { - true => (size, 0), - false => (size, size), - } - }) - .unwrap_or((0, 0)); + .map(|file_cache| file_cache.config.calculate_cache_size(usable_system_memory)) + .unwrap_or(0); if let Some(cgroup) = &self.cgroup { let (last_time, last_history) = *cgroup.watcher.borrow(); @@ -273,7 +261,7 @@ impl Runner { let new_threshold = self .config - .cgroup_threshold(usable_system_memory, expected_file_cache_disk_size); + .cgroup_threshold(usable_system_memory, expected_file_cache_size); let current = last_history.avg_non_reclaimable; @@ -300,13 +288,10 @@ impl Runner { .set_file_cache_size(expected_file_cache_size) .await .context("failed to set file cache size")?; - if !file_cache.config.in_memory { - file_cache_disk_size = actual_usage; - } + file_cache_disk_size = actual_usage; let message = format!( - "set file cache size to {} MiB (in memory = {})", + "set file cache size to {} MiB", bytes_to_mebibytes(actual_usage), - file_cache.config.in_memory, ); info!("downscale: {message}"); status.push(message); @@ -357,9 +342,7 @@ impl Runner { .set_file_cache_size(expected_usage) .await .context("failed to set file cache size")?; - if !file_cache.config.in_memory { - file_cache_disk_size = actual_usage; - } + file_cache_disk_size = actual_usage; if actual_usage != expected_usage { warn!( From 4c7fa12a2aa1967329c29aafe1341e01460ed032 Mon Sep 17 00:00:00 2001 From: khanova <32508607+khanova@users.noreply.github.com> Date: Thu, 2 Nov 2023 17:26:15 +0100 Subject: [PATCH 67/69] Proxy introduce allowed ips (#5729) ## Problem Proxy doesn't accept wake_compute responses with the allowed IPs. ## Summary of changes Extend wake_compute api to be able to return allowed_ips. --- proxy/src/console/messages.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/proxy/src/console/messages.rs b/proxy/src/console/messages.rs index 0d321c077a..e5f1615b14 100644 --- a/proxy/src/console/messages.rs +++ b/proxy/src/console/messages.rs @@ -13,6 +13,7 @@ pub struct ConsoleError { #[derive(Deserialize)] pub struct GetRoleSecret { pub role_secret: Box, + pub allowed_ips: Option>>, } // Manually implement debug to omit sensitive info. @@ -187,4 +188,31 @@ mod tests { Ok(()) } + + #[test] + fn parse_wake_compute() -> anyhow::Result<()> { + let json = json!({ + "address": "0.0.0.0", + "aux": dummy_aux(), + }); + let _: WakeCompute = serde_json::from_str(&json.to_string())?; + Ok(()) + } + + #[test] + fn parse_get_role_secret() -> anyhow::Result<()> { + // Empty `allowed_ips` field. + let json = json!({ + "role_secret": "secret", + }); + let _: GetRoleSecret = serde_json::from_str(&json.to_string())?; + // Empty `allowed_ips` field. + let json = json!({ + "role_secret": "secret", + "allowed_ips": ["8.8.8.8"], + }); + let _: GetRoleSecret = serde_json::from_str(&json.to_string())?; + + Ok(()) + } } From 27bdbf5e365bc77d5a7406e9fefa76e9a31ec12b Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 2 Nov 2023 21:05:33 +0200 Subject: [PATCH 68/69] chore(layer): restore logging, doc changes (#5766) Some of the log messages were lost with the #4938. This PR adds some of them back, most notably: - starting to on-demand download - successful completion of on-demand download - ability to see when there were many waiters for the layer download - "unexpectedly on-demand downloading ..." is now `info!` Additionally some rare events are logged as error, which should never happen. --- libs/utils/src/sync/heavier_once_cell.rs | 41 ++++++++++++++++++-- pageserver/src/tenant/storage_layer/layer.rs | 40 ++++++++++++++----- 2 files changed, 67 insertions(+), 14 deletions(-) diff --git a/libs/utils/src/sync/heavier_once_cell.rs b/libs/utils/src/sync/heavier_once_cell.rs index 8a5aaf2ceb..0ccaf4e716 100644 --- a/libs/utils/src/sync/heavier_once_cell.rs +++ b/libs/utils/src/sync/heavier_once_cell.rs @@ -1,4 +1,7 @@ -use std::sync::{Arc, Mutex, MutexGuard}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, Mutex, MutexGuard, +}; use tokio::sync::Semaphore; /// Custom design like [`tokio::sync::OnceCell`] but using [`OwnedSemaphorePermit`] instead of @@ -10,6 +13,7 @@ use tokio::sync::Semaphore; /// [`OwnedSemaphorePermit`]: tokio::sync::OwnedSemaphorePermit pub struct OnceCell { inner: Mutex>, + initializers: AtomicUsize, } impl Default for OnceCell { @@ -17,6 +21,7 @@ impl Default for OnceCell { fn default() -> Self { Self { inner: Default::default(), + initializers: AtomicUsize::new(0), } } } @@ -49,6 +54,7 @@ impl OnceCell { init_semaphore: Arc::new(sem), value: Some(value), }), + initializers: AtomicUsize::new(0), } } @@ -71,7 +77,11 @@ impl OnceCell { guard.init_semaphore.clone() }; - let permit = sem.acquire_owned().await; + let permit = { + // increment the count for the duration of queued + let _guard = CountWaitingInitializers::start(self); + sem.acquire_owned().await + }; match permit { Ok(permit) => { @@ -100,12 +110,13 @@ impl OnceCell { /// /// If the inner has already been initialized. pub fn set(&self, value: T, _permit: InitPermit) -> Guard<'_, T> { - // cannot assert that this permit is for self.inner.semaphore let guard = self.inner.lock().unwrap(); + // cannot assert that this permit is for self.inner.semaphore, but we can assert it cannot + // give more permits right now. if guard.init_semaphore.try_acquire().is_ok() { drop(guard); - panic!("semaphore is of wrong origin"); + panic!("permit is of wrong origin"); } Self::set0(value, guard) @@ -130,6 +141,28 @@ impl OnceCell { None } } + + /// Return the number of [`Self::get_or_init`] calls waiting for initialization to complete. + pub fn initializer_count(&self) -> usize { + self.initializers.load(Ordering::Relaxed) + } +} + +/// DropGuard counter for queued tasks waiting to initialize, mainly accessible for the +/// initializing task for example at the end of initialization. +struct CountWaitingInitializers<'a, T>(&'a OnceCell); + +impl<'a, T> CountWaitingInitializers<'a, T> { + fn start(target: &'a OnceCell) -> Self { + target.initializers.fetch_add(1, Ordering::Relaxed); + CountWaitingInitializers(target) + } +} + +impl<'a, T> Drop for CountWaitingInitializers<'a, T> { + fn drop(&mut self) { + self.0.initializers.fetch_sub(1, Ordering::Relaxed); + } } /// Uninteresting guard object to allow short-lived access to inspect or clone the held, diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index b320c02f9b..d72982a9a0 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -411,6 +411,10 @@ struct LayerInner { version: AtomicUsize, /// Allow subscribing to when the layer actually gets evicted. + /// + /// If in future we need to implement "wait until layer instances are gone and done", carrying + /// this over to the gc spawn_blocking from LayerInner::drop will do the trick, and adding a + /// method for "wait_gc" which will wait to this being closed. status: tokio::sync::broadcast::Sender, /// Counter for exponential backoff with the download @@ -561,6 +565,8 @@ impl LayerInner { } } + /// Cancellation safe, however dropping the future and calling this method again might result + /// in a new attempt to evict OR join the previously started attempt. pub(crate) async fn evict_and_wait( &self, _: &RemoteTimelineClient, @@ -609,8 +615,8 @@ impl LayerInner { } } - /// Should be cancellation safe, but cancellation is troublesome together with the spawned - /// download. + /// Cancellation safe. + #[tracing::instrument(skip_all, fields(layer=%self))] async fn get_or_maybe_download( self: &Arc, allow_download: bool, @@ -654,8 +660,6 @@ impl LayerInner { return Err(DownloadError::NoRemoteStorage); } - tracing::debug!(%reason, "downloading layer"); - if let Some(ctx) = ctx { self.check_expected_download(ctx)?; } @@ -666,6 +670,8 @@ impl LayerInner { return Err(DownloadError::DownloadRequired); } + tracing::info!(%reason, "downloading on-demand"); + self.spawn_download_and_wait(timeline, permit).await? } else { // the file is present locally, probably by a previous but cancelled call to @@ -686,6 +692,11 @@ impl LayerInner { LayerResidenceEventReason::ResidenceChange, ); + let waiters = self.inner.initializer_count(); + if waiters > 0 { + tracing::info!(waiters, "completing the on-demand download for other tasks"); + } + Ok((ResidentOrWantedEvicted::Resident(res), permit)) }; @@ -746,8 +757,8 @@ impl LayerInner { match b { Download => Ok(()), Warn | Error => { - tracing::warn!( - "unexpectedly on-demand downloading remote layer {self} for task kind {:?}", + tracing::info!( + "unexpectedly on-demand downloading for task kind {:?}", ctx.task_kind() ); crate::metrics::UNEXPECTED_ONDEMAND_DOWNLOADS.inc(); @@ -779,6 +790,7 @@ impl LayerInner { // block tenant::mgr::remove_tenant_from_memory. let this: Arc = self.clone(); + crate::task_mgr::spawn( &tokio::runtime::Handle::current(), crate::task_mgr::TaskKind::RemoteDownloadTask, @@ -787,6 +799,7 @@ impl LayerInner { &task_name, false, async move { + let client = timeline .remote_client .as_ref() @@ -848,6 +861,7 @@ impl LayerInner { } self.consecutive_failures.store(0, Ordering::Relaxed); + tracing::info!("on-demand download successful"); Ok(permit) } @@ -1040,11 +1054,14 @@ impl LayerInner { Ok(()) } Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - tracing::info!("failed to evict file from disk, it was already gone"); + tracing::error!( + layer_size = %self.desc.file_size, + "failed to evict layer from disk, it was already gone (metrics will be inaccurate)" + ); Err(EvictionCancelled::FileNotFound) } Err(e) => { - tracing::warn!("failed to evict file from disk: {e:#}"); + tracing::error!("failed to evict file from disk: {e:#}"); Err(EvictionCancelled::RemoveFailed) } }; @@ -1124,6 +1141,8 @@ impl std::fmt::Display for NeedsDownload { /// Existence of `DownloadedLayer` means that we have the file locally, and can later evict it. pub(crate) struct DownloadedLayer { owner: Weak, + // Use tokio OnceCell as we do not need to deinitialize this, it'll just get dropped with the + // DownloadedLayer kind: tokio::sync::OnceCell>, version: usize, } @@ -1167,7 +1186,6 @@ impl DownloadedLayer { "these are the same, just avoiding the upgrade" ); - // there is nothing async here, but it should be async let res = if owner.desc.is_delta { let summary = Some(delta_layer::Summary::expected( owner.desc.tenant_id, @@ -1266,6 +1284,8 @@ impl std::fmt::Debug for ResidentLayer { impl ResidentLayer { /// Release the eviction guard, converting back into a plain [`Layer`]. + /// + /// You can access the [`Layer`] also by using `as_ref`. pub(crate) fn drop_eviction_guard(self) -> Layer { self.into() } @@ -1321,7 +1341,7 @@ impl AsRef for ResidentLayer { } } -/// Allow slimming down if we don't want the `2*usize` with eviction candidates? +/// Drop the eviction guard. impl From for Layer { fn from(value: ResidentLayer) -> Self { value.owner From cdcaa329bfb9dd045a20b5012b3f4e97f591ce7c Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Fri, 3 Nov 2023 08:30:58 +0000 Subject: [PATCH 69/69] proxy: no more statements (#5747) ## Problem my prepared statements change in tokio-postgres landed in the latest release. it didn't work as we intended ## Summary of changes https://github.com/neondatabase/rust-postgres/pull/24 --- Cargo.lock | 10 ++-- Cargo.toml | 12 ++--- proxy/src/serverless/sql_over_http.rs | 68 ++++++++++++++------------- test_runner/regress/test_proxy.py | 44 +++++++++++++++++ 4 files changed, 90 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 39c33304ae..4d9b20dae4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3550,7 +3550,7 @@ dependencies = [ [[package]] name = "postgres" version = "0.19.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=7434d9388965a17a6d113e5dfc0e65666a03b4c2#7434d9388965a17a6d113e5dfc0e65666a03b4c2" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=ce7260db5998fe27167da42503905a12e7ad9048#ce7260db5998fe27167da42503905a12e7ad9048" dependencies = [ "bytes", "fallible-iterator", @@ -3563,7 +3563,7 @@ dependencies = [ [[package]] name = "postgres-native-tls" version = "0.5.0" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=7434d9388965a17a6d113e5dfc0e65666a03b4c2#7434d9388965a17a6d113e5dfc0e65666a03b4c2" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=ce7260db5998fe27167da42503905a12e7ad9048#ce7260db5998fe27167da42503905a12e7ad9048" dependencies = [ "native-tls", "tokio", @@ -3574,7 +3574,7 @@ dependencies = [ [[package]] name = "postgres-protocol" version = "0.6.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=7434d9388965a17a6d113e5dfc0e65666a03b4c2#7434d9388965a17a6d113e5dfc0e65666a03b4c2" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=ce7260db5998fe27167da42503905a12e7ad9048#ce7260db5998fe27167da42503905a12e7ad9048" dependencies = [ "base64 0.20.0", "byteorder", @@ -3592,7 +3592,7 @@ dependencies = [ [[package]] name = "postgres-types" version = "0.2.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=7434d9388965a17a6d113e5dfc0e65666a03b4c2#7434d9388965a17a6d113e5dfc0e65666a03b4c2" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=ce7260db5998fe27167da42503905a12e7ad9048#ce7260db5998fe27167da42503905a12e7ad9048" dependencies = [ "bytes", "fallible-iterator", @@ -5396,7 +5396,7 @@ dependencies = [ [[package]] name = "tokio-postgres" version = "0.7.7" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=7434d9388965a17a6d113e5dfc0e65666a03b4c2#7434d9388965a17a6d113e5dfc0e65666a03b4c2" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=ce7260db5998fe27167da42503905a12e7ad9048#ce7260db5998fe27167da42503905a12e7ad9048" dependencies = [ "async-trait", "byteorder", diff --git a/Cargo.toml b/Cargo.toml index a0be7bb9ac..6f538941f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -161,11 +161,11 @@ env_logger = "0.10" log = "0.4" ## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed -postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } -postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } -postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } -postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } +postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } +postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } +postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } +postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } ## Other git libraries heapless = { default-features=false, features=[], git = "https://github.com/japaric/heapless.git", rev = "644653bf3b831c6bb4963be2de24804acf5e5001" } # upstream release pending @@ -202,7 +202,7 @@ tonic-build = "0.9" # This is only needed for proxy's tests. # TODO: we should probably fork `tokio-postgres-rustls` instead. -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="ce7260db5998fe27167da42503905a12e7ad9048" } ################# Binary contents sections diff --git a/proxy/src/serverless/sql_over_http.rs b/proxy/src/serverless/sql_over_http.rs index 93df86cfc4..8f7cf7fbaf 100644 --- a/proxy/src/serverless/sql_over_http.rs +++ b/proxy/src/serverless/sql_over_http.rs @@ -470,30 +470,26 @@ async fn query_to_json( } .and_then(|s| s.parse::().ok()); - let fields = if !rows.is_empty() { - rows[0] - .columns() - .iter() - .map(|c| { - json!({ - "name": Value::String(c.name().to_owned()), - "dataTypeID": Value::Number(c.type_().oid().into()), - "tableID": c.table_oid(), - "columnID": c.column_id(), - "dataTypeSize": c.type_size(), - "dataTypeModifier": c.type_modifier(), - "format": "text", - }) - }) - .collect::>() - } else { - Vec::new() - }; + let mut fields = vec![]; + let mut columns = vec![]; + + for c in row_stream.columns() { + fields.push(json!({ + "name": Value::String(c.name().to_owned()), + "dataTypeID": Value::Number(c.type_().oid().into()), + "tableID": c.table_oid(), + "columnID": c.column_id(), + "dataTypeSize": c.type_size(), + "dataTypeModifier": c.type_modifier(), + "format": "text", + })); + columns.push(client.get_type(c.type_oid()).await?); + } // convert rows to JSON let rows = rows .iter() - .map(|row| pg_text_row_to_json(row, raw_output, array_mode)) + .map(|row| pg_text_row_to_json(row, &columns, raw_output, array_mode)) .collect::, _>>()?; // resulting JSON format is based on the format of node-postgres result @@ -514,22 +510,28 @@ async fn query_to_json( // pub fn pg_text_row_to_json( row: &Row, + columns: &[Type], raw_output: bool, array_mode: bool, ) -> Result { - let iter = row.columns().iter().enumerate().map(|(i, column)| { - let name = column.name(); - let pg_value = row.as_text(i)?; - let json_value = if raw_output { - match pg_value { - Some(v) => Value::String(v.to_string()), - None => Value::Null, - } - } else { - pg_text_to_json(pg_value, column.type_())? - }; - Ok((name.to_string(), json_value)) - }); + let iter = row + .columns() + .iter() + .zip(columns) + .enumerate() + .map(|(i, (column, typ))| { + let name = column.name(); + let pg_value = row.as_text(i)?; + let json_value = if raw_output { + match pg_value { + Some(v) => Value::String(v.to_string()), + None => Value::Null, + } + } else { + pg_text_to_json(pg_value, typ)? + }; + Ok((name.to_string(), json_value)) + }); if array_mode { // drop keys and aggregate into array diff --git a/test_runner/regress/test_proxy.py b/test_runner/regress/test_proxy.py index 6388e9f365..c93cdf637a 100644 --- a/test_runner/regress/test_proxy.py +++ b/test_runner/regress/test_proxy.py @@ -432,3 +432,47 @@ def test_sql_over_http_pool_idle(static_proxy: NeonProxy): query(200, "BEGIN") pid2 = query(200, GET_CONNECTION_PID_QUERY)["rows"][0]["pid"] assert pid1 != pid2 + + +@pytest.mark.timeout(60) +def test_sql_over_http_pool_dos(static_proxy: NeonProxy): + static_proxy.safe_psql("create user http_auth with password 'http' superuser") + + static_proxy.safe_psql("CREATE TYPE foo AS ENUM ('foo')") + + def query(status: int, query: str) -> Any: + return static_proxy.http_query( + query, + [], + user="http_auth", + password="http", + expected_code=status, + ) + + # query generates a million rows - should hit the 10MB reponse limit quickly + response = query( + 400, + "select * from generate_series(1, 5000) a cross join generate_series(1, 5000) b cross join (select 'foo'::foo) c;", + ) + assert "response is too large (max is 10485760 bytes)" in response["message"] + + +def test_sql_over_http_pool_custom_types(static_proxy: NeonProxy): + static_proxy.safe_psql("create user http_auth with password 'http' superuser") + + static_proxy.safe_psql("CREATE TYPE foo AS ENUM ('foo','bar','baz')") + + def query(status: int, query: str) -> Any: + return static_proxy.http_query( + query, + [], + user="http_auth", + password="http", + expected_code=status, + ) + + response = query( + 200, + "select array['foo'::foo, 'bar'::foo, 'baz'::foo] as data", + ) + assert response["rows"][0]["data"] == ["foo", "bar", "baz"]