From 7131ac4730f7268a8624a9c7345c23938cc8b6a6 Mon Sep 17 00:00:00 2001 From: Yuchen Liang <70461588+yliang412@users.noreply.github.com> Date: Sat, 17 Aug 2024 11:48:53 -0400 Subject: [PATCH 1/6] refactor(scrubber): add unified command suitable for cron job (#8635) Part of #8128. ## Description This PR creates a unified command to run both physical gc and metadata health check as a cron job. This also enables us to add additional tasks to the cron job in the future. Signed-off-by: Yuchen Liang --- storage_scrubber/src/main.rs | 240 ++++++++++++------ .../src/pageserver_physical_gc.rs | 16 +- .../src/scan_pageserver_metadata.rs | 2 +- 3 files changed, 175 insertions(+), 83 deletions(-) diff --git a/storage_scrubber/src/main.rs b/storage_scrubber/src/main.rs index cbc836755a..3935e513e3 100644 --- a/storage_scrubber/src/main.rs +++ b/storage_scrubber/src/main.rs @@ -3,9 +3,10 @@ use camino::Utf8PathBuf; use pageserver_api::controller_api::{MetadataHealthUpdateRequest, MetadataHealthUpdateResponse}; use pageserver_api::shard::TenantShardId; use reqwest::{Method, Url}; +use storage_controller_client::control_api; use storage_scrubber::garbage::{find_garbage, purge_garbage, PurgeMode}; use storage_scrubber::pageserver_physical_gc::GcMode; -use storage_scrubber::scan_pageserver_metadata::scan_metadata; +use storage_scrubber::scan_pageserver_metadata::scan_pageserver_metadata; use storage_scrubber::tenant_snapshot::SnapshotDownloader; use storage_scrubber::{find_large_objects, ControllerClientConfig}; use storage_scrubber::{ @@ -68,7 +69,7 @@ enum Command { #[arg(long = "tenant-id", num_args = 0..)] tenant_ids: Vec, #[arg(long = "post", default_value_t = false)] - post_to_storage_controller: bool, + post_to_storcon: bool, #[arg(long, default_value = None)] /// For safekeeper node_kind only, points to db with debug dump dump_db_connstr: Option, @@ -100,6 +101,16 @@ enum Command { #[arg(long = "concurrency", short = 'j', default_value_t = 64)] concurrency: usize, }, + CronJob { + // PageserverPhysicalGc + #[arg(long = "min-age")] + gc_min_age: humantime::Duration, + #[arg(short, long, default_value_t = GcMode::IndicesOnly)] + gc_mode: GcMode, + // ScanMetadata + #[arg(long = "post", default_value_t = false)] + post_to_storcon: bool, + }, } #[tokio::main] @@ -117,6 +128,7 @@ async fn main() -> anyhow::Result<()> { Command::TenantSnapshot { .. } => "tenant-snapshot", Command::PageserverPhysicalGc { .. } => "pageserver-physical-gc", Command::FindLargeObjects { .. } => "find-large-objects", + Command::CronJob { .. } => "cron-job", }; let _guard = init_logging(&format!( "{}_{}_{}_{}.log", @@ -126,12 +138,13 @@ async fn main() -> anyhow::Result<()> { chrono::Utc::now().format("%Y_%m_%d__%H_%M_%S") )); - let controller_client_conf = cli.controller_api.map(|controller_api| { + let controller_client = cli.controller_api.map(|controller_api| { ControllerClientConfig { controller_api, // Default to no key: this is a convenience when working in a development environment controller_jwt: cli.controller_jwt.unwrap_or("".to_owned()), } + .build_client() }); match cli.command { @@ -139,7 +152,7 @@ async fn main() -> anyhow::Result<()> { json, tenant_ids, node_kind, - post_to_storage_controller, + post_to_storcon, dump_db_connstr, dump_db_table, } => { @@ -178,53 +191,14 @@ async fn main() -> anyhow::Result<()> { } Ok(()) } else { - if controller_client_conf.is_none() && post_to_storage_controller { - return Err(anyhow!("Posting pageserver scan health status to storage controller requires `--controller-api` and `--controller-jwt` to run")); - } - match scan_metadata(bucket_config.clone(), tenant_ids).await { - Err(e) => { - tracing::error!("Failed: {e}"); - Err(e) - } - Ok(summary) => { - if json { - println!("{}", serde_json::to_string(&summary).unwrap()) - } else { - println!("{}", summary.summary_string()); - } - - if post_to_storage_controller { - if let Some(conf) = controller_client_conf { - let controller_client = conf.build_client(); - let body = summary.build_health_update_request(); - controller_client - .dispatch::( - Method::POST, - "control/v1/metadata_health/update".to_string(), - Some(body), - ) - .await?; - } - } - - if summary.is_fatal() { - tracing::error!("Fatal scrub errors detected"); - } else if summary.is_empty() { - // Strictly speaking an empty bucket is a valid bucket, but if someone ran the - // scrubber they were likely expecting to scan something, and if we see no timelines - // at all then it's likely due to some configuration issues like a bad prefix - tracing::error!( - "No timelines found in bucket {} prefix {}", - bucket_config.bucket, - bucket_config - .prefix_in_bucket - .unwrap_or("".to_string()) - ); - } - - Ok(()) - } - } + scan_pageserver_metadata_cmd( + bucket_config, + controller_client.as_ref(), + tenant_ids, + json, + post_to_storcon, + ) + .await } } Command::FindGarbage { @@ -254,31 +228,14 @@ async fn main() -> anyhow::Result<()> { min_age, mode, } => { - match (&controller_client_conf, mode) { - (Some(_), _) => { - // Any mode may run when controller API is set - } - (None, GcMode::Full) => { - // The part of physical GC where we erase ancestor layers cannot be done safely without - // confirming the most recent complete shard split with the controller. Refuse to run, rather - // than doing it unsafely. - return Err(anyhow!("Full physical GC requires `--controller-api` and `--controller-jwt` to run")); - } - (None, GcMode::DryRun | GcMode::IndicesOnly) => { - // These GcModes do not require the controller to run. - } - } - - let summary = pageserver_physical_gc( - bucket_config, - controller_client_conf, + pageserver_physical_gc_cmd( + &bucket_config, + controller_client.as_ref(), tenant_ids, - min_age.into(), + min_age, mode, ) - .await?; - println!("{}", serde_json::to_string(&summary).unwrap()); - Ok(()) + .await } Command::FindLargeObjects { min_size, @@ -295,5 +252,142 @@ async fn main() -> anyhow::Result<()> { println!("{}", serde_json::to_string(&summary).unwrap()); Ok(()) } + Command::CronJob { + gc_min_age, + gc_mode, + post_to_storcon, + } => { + run_cron_job( + bucket_config, + controller_client.as_ref(), + gc_min_age, + gc_mode, + post_to_storcon, + ) + .await + } + } +} + +/// Runs the scrubber cron job. +/// 1. Do pageserver physical gc +/// 2. Scan pageserver metadata +pub async fn run_cron_job( + bucket_config: BucketConfig, + controller_client: Option<&control_api::Client>, + gc_min_age: humantime::Duration, + gc_mode: GcMode, + post_to_storcon: bool, +) -> anyhow::Result<()> { + tracing::info!(%gc_min_age, %gc_mode, "Running pageserver-physical-gc"); + pageserver_physical_gc_cmd( + &bucket_config, + controller_client, + Vec::new(), + gc_min_age, + gc_mode, + ) + .await?; + tracing::info!(%post_to_storcon, node_kind = %NodeKind::Pageserver, "Running scan-metadata"); + scan_pageserver_metadata_cmd( + bucket_config, + controller_client, + Vec::new(), + true, + post_to_storcon, + ) + .await?; + + Ok(()) +} + +pub async fn pageserver_physical_gc_cmd( + bucket_config: &BucketConfig, + controller_client: Option<&control_api::Client>, + tenant_shard_ids: Vec, + min_age: humantime::Duration, + mode: GcMode, +) -> anyhow::Result<()> { + match (controller_client, mode) { + (Some(_), _) => { + // Any mode may run when controller API is set + } + (None, GcMode::Full) => { + // The part of physical GC where we erase ancestor layers cannot be done safely without + // confirming the most recent complete shard split with the controller. Refuse to run, rather + // than doing it unsafely. + return Err(anyhow!( + "Full physical GC requires `--controller-api` and `--controller-jwt` to run" + )); + } + (None, GcMode::DryRun | GcMode::IndicesOnly) => { + // These GcModes do not require the controller to run. + } + } + + let summary = pageserver_physical_gc( + bucket_config, + controller_client, + tenant_shard_ids, + min_age.into(), + mode, + ) + .await?; + println!("{}", serde_json::to_string(&summary).unwrap()); + Ok(()) +} + +pub async fn scan_pageserver_metadata_cmd( + bucket_config: BucketConfig, + controller_client: Option<&control_api::Client>, + tenant_shard_ids: Vec, + json: bool, + post_to_storcon: bool, +) -> anyhow::Result<()> { + if controller_client.is_none() && post_to_storcon { + return Err(anyhow!("Posting pageserver scan health status to storage controller requires `--controller-api` and `--controller-jwt` to run")); + } + match scan_pageserver_metadata(bucket_config.clone(), tenant_shard_ids).await { + Err(e) => { + tracing::error!("Failed: {e}"); + Err(e) + } + Ok(summary) => { + if json { + println!("{}", serde_json::to_string(&summary).unwrap()) + } else { + println!("{}", summary.summary_string()); + } + + if post_to_storcon { + if let Some(client) = controller_client { + let body = summary.build_health_update_request(); + client + .dispatch::( + Method::POST, + "control/v1/metadata_health/update".to_string(), + Some(body), + ) + .await?; + } + } + + if summary.is_fatal() { + tracing::error!("Fatal scrub errors detected"); + } else if summary.is_empty() { + // Strictly speaking an empty bucket is a valid bucket, but if someone ran the + // scrubber they were likely expecting to scan something, and if we see no timelines + // at all then it's likely due to some configuration issues like a bad prefix + tracing::error!( + "No timelines found in bucket {} prefix {}", + bucket_config.bucket, + bucket_config + .prefix_in_bucket + .unwrap_or("".to_string()) + ); + } + + Ok(()) + } } } diff --git a/storage_scrubber/src/pageserver_physical_gc.rs b/storage_scrubber/src/pageserver_physical_gc.rs index c8b1ed49f4..20d9bd6dd4 100644 --- a/storage_scrubber/src/pageserver_physical_gc.rs +++ b/storage_scrubber/src/pageserver_physical_gc.rs @@ -4,9 +4,7 @@ use std::time::{Duration, SystemTime}; use crate::checks::{list_timeline_blobs, BlobDataParseResult}; use crate::metadata_stream::{stream_tenant_timelines, stream_tenants}; -use crate::{ - init_remote, BucketConfig, ControllerClientConfig, NodeKind, RootTarget, TenantShardTimelineId, -}; +use crate::{init_remote, BucketConfig, NodeKind, RootTarget, TenantShardTimelineId}; use aws_sdk_s3::Client; use futures_util::{StreamExt, TryStreamExt}; use pageserver::tenant::remote_timeline_client::index::LayerFileMetadata; @@ -473,8 +471,8 @@ async fn gc_ancestor( /// This type of GC is not necessary for correctness: rather it serves to reduce wasted storage capacity, and /// make sure that object listings don't get slowed down by large numbers of garbage objects. pub async fn pageserver_physical_gc( - bucket_config: BucketConfig, - controller_client_conf: Option, + bucket_config: &BucketConfig, + controller_client: Option<&control_api::Client>, tenant_shard_ids: Vec, min_age: Duration, mode: GcMode, @@ -558,7 +556,7 @@ pub async fn pageserver_physical_gc( let timelines = timelines.map_ok(|ttid| { gc_timeline( &s3_client, - &bucket_config, + bucket_config, &min_age, &target, mode, @@ -574,7 +572,7 @@ pub async fn pageserver_physical_gc( } // Execute cross-shard GC, using the accumulator's full view of all the shards built in the per-shard GC - let Some(controller_client) = controller_client_conf.map(|c| c.build_client()) else { + let Some(client) = controller_client else { tracing::info!("Skipping ancestor layer GC, because no `--controller-api` was specified"); return Ok(summary); }; @@ -583,13 +581,13 @@ pub async fn pageserver_physical_gc( .unwrap() .into_inner() .unwrap() - .into_gc_ancestors(&controller_client, &mut summary) + .into_gc_ancestors(client, &mut summary) .await; for ancestor_shard in ancestor_shards { gc_ancestor( &s3_client, - &bucket_config, + bucket_config, &target, &min_age, ancestor_shard, diff --git a/storage_scrubber/src/scan_pageserver_metadata.rs b/storage_scrubber/src/scan_pageserver_metadata.rs index b9630056e1..2409b7b132 100644 --- a/storage_scrubber/src/scan_pageserver_metadata.rs +++ b/storage_scrubber/src/scan_pageserver_metadata.rs @@ -116,7 +116,7 @@ Index versions: {version_summary} } /// Scan the pageserver metadata in an S3 bucket, reporting errors and statistics. -pub async fn scan_metadata( +pub async fn scan_pageserver_metadata( bucket_config: BucketConfig, tenant_ids: Vec, ) -> anyhow::Result { From 188bde7f0776636310260cbf636922d1029add7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Sun, 18 Aug 2024 19:32:10 +0200 Subject: [PATCH 2/6] Default image compression to zstd at level 1 (#8677) After the rollout has succeeded, we now set the default image compression to be enabled. We also remove its explicit mention from `neon_fixtures.py` added in #8368 as it is now the default (and we switch to `zstd(1)` which is a bit nicer on CPU time). Part of https://github.com/neondatabase/neon/issues/5431 --- pageserver/src/config.rs | 12 +++++------- pageserver/src/tenant/storage_layer/split_writer.rs | 7 ++++++- test_runner/fixtures/neon_fixtures.py | 1 - 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index 3ac5ac539f..0ebaf78840 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -50,7 +50,6 @@ pub mod defaults { DEFAULT_HTTP_LISTEN_ADDR, DEFAULT_HTTP_LISTEN_PORT, DEFAULT_PG_LISTEN_ADDR, DEFAULT_PG_LISTEN_PORT, }; - use pageserver_api::models::ImageCompressionAlgorithm; pub use storage_broker::DEFAULT_ENDPOINT as BROKER_DEFAULT_ENDPOINT; pub const DEFAULT_WAIT_LSN_TIMEOUT: &str = "300 s"; @@ -90,8 +89,7 @@ pub mod defaults { pub const DEFAULT_MAX_VECTORED_READ_BYTES: usize = 128 * 1024; // 128 KiB - pub const DEFAULT_IMAGE_COMPRESSION: ImageCompressionAlgorithm = - ImageCompressionAlgorithm::Disabled; + pub const DEFAULT_IMAGE_COMPRESSION: &str = "zstd(1)"; pub const DEFAULT_VALIDATE_VECTORED_GET: bool = false; @@ -478,7 +476,7 @@ impl PageServerConfigBuilder { max_vectored_read_bytes: Set(MaxVectoredReadBytes( NonZeroUsize::new(DEFAULT_MAX_VECTORED_READ_BYTES).unwrap(), )), - image_compression: Set(DEFAULT_IMAGE_COMPRESSION), + image_compression: Set(DEFAULT_IMAGE_COMPRESSION.parse().unwrap()), ephemeral_bytes_per_memory_kb: Set(DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB), l0_flush: Set(L0FlushConfig::default()), compact_level0_phase1_value_access: Set(CompactL0Phase1ValueAccess::default()), @@ -1065,7 +1063,7 @@ impl PageServerConf { NonZeroUsize::new(defaults::DEFAULT_MAX_VECTORED_READ_BYTES) .expect("Invalid default constant"), ), - image_compression: defaults::DEFAULT_IMAGE_COMPRESSION, + image_compression: defaults::DEFAULT_IMAGE_COMPRESSION.parse().unwrap(), ephemeral_bytes_per_memory_kb: defaults::DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB, l0_flush: L0FlushConfig::default(), compact_level0_phase1_value_access: CompactL0Phase1ValueAccess::default(), @@ -1305,7 +1303,7 @@ background_task_maximum_delay = '334 s' NonZeroUsize::new(defaults::DEFAULT_MAX_VECTORED_READ_BYTES) .expect("Invalid default constant") ), - image_compression: defaults::DEFAULT_IMAGE_COMPRESSION, + image_compression: defaults::DEFAULT_IMAGE_COMPRESSION.parse().unwrap(), ephemeral_bytes_per_memory_kb: defaults::DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB, l0_flush: L0FlushConfig::default(), compact_level0_phase1_value_access: CompactL0Phase1ValueAccess::default(), @@ -1378,7 +1376,7 @@ background_task_maximum_delay = '334 s' NonZeroUsize::new(defaults::DEFAULT_MAX_VECTORED_READ_BYTES) .expect("Invalid default constant") ), - image_compression: defaults::DEFAULT_IMAGE_COMPRESSION, + image_compression: defaults::DEFAULT_IMAGE_COMPRESSION.parse().unwrap(), ephemeral_bytes_per_memory_kb: defaults::DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB, l0_flush: L0FlushConfig::default(), compact_level0_phase1_value_access: CompactL0Phase1ValueAccess::default(), diff --git a/pageserver/src/tenant/storage_layer/split_writer.rs b/pageserver/src/tenant/storage_layer/split_writer.rs index d7bfe48c60..e12e29cd45 100644 --- a/pageserver/src/tenant/storage_layer/split_writer.rs +++ b/pageserver/src/tenant/storage_layer/split_writer.rs @@ -208,6 +208,8 @@ impl SplitDeltaLayerWriter { #[cfg(test)] mod tests { + use rand::{RngCore, SeedableRng}; + use crate::{ tenant::{ harness::{TenantHarness, TIMELINE_ID}, @@ -229,7 +231,10 @@ mod tests { } fn get_large_img() -> Bytes { - vec![0; 8192].into() + let mut rng = rand::rngs::SmallRng::seed_from_u64(42); + let mut data = vec![0; 8192]; + rng.fill_bytes(&mut data); + data.into() } #[tokio::test] diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index ec5a83601e..ba6fbc003a 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1162,7 +1162,6 @@ class NeonEnv: "listen_http_addr": f"localhost:{pageserver_port.http}", "pg_auth_type": pg_auth_type, "http_auth_type": http_auth_type, - "image_compression": "zstd", } if self.pageserver_virtual_file_io_engine is not None: ps_cfg["virtual_file_io_engine"] = self.pageserver_virtual_file_io_engine From f246aa3ca7f19993e0582dfd8069375c09c5158c Mon Sep 17 00:00:00 2001 From: Folke Behrens Date: Mon, 19 Aug 2024 10:33:46 +0200 Subject: [PATCH 3/6] proxy: Fix some warnings by extended clippy checks (#8748) * Missing blank lifetimes which is now deprecated. * Matching off unqualified enum variants that could act like variable. * Missing semicolons. --- proxy/src/auth.rs | 46 +++-- proxy/src/auth/backend.rs | 72 +++----- proxy/src/auth/backend/jwt.rs | 6 +- proxy/src/auth/credentials.rs | 17 +- proxy/src/cache/common.rs | 2 +- proxy/src/cache/timed_lru.rs | 2 +- proxy/src/compute.rs | 23 ++- proxy/src/config.rs | 6 +- proxy/src/console/messages.rs | 22 ++- proxy/src/console/mgmt.rs | 3 +- proxy/src/console/provider.rs | 170 +++++++++--------- proxy/src/context.rs | 8 +- proxy/src/context/parquet.rs | 2 +- proxy/src/intern.rs | 2 +- proxy/src/metrics.rs | 2 +- proxy/src/proxy/copy_bidirectional.rs | 2 +- proxy/src/proxy/handshake.rs | 15 +- proxy/src/proxy/tests/mitm.rs | 4 +- proxy/src/rate_limiter/limit_algorithm.rs | 4 +- .../src/rate_limiter/limit_algorithm/aimd.rs | 5 +- .../connection_with_credentials_provider.rs | 2 +- proxy/src/redis/notifications.rs | 23 ++- proxy/src/sasl.rs | 5 +- proxy/src/sasl/channel_binding.rs | 21 +-- proxy/src/sasl/messages.rs | 5 +- proxy/src/scram.rs | 4 +- proxy/src/scram/countmin.rs | 2 - proxy/src/scram/exchange.rs | 22 +-- proxy/src/scram/messages.rs | 6 +- proxy/src/scram/pbkdf2.rs | 2 +- proxy/src/scram/threadpool.rs | 4 +- proxy/src/serverless.rs | 4 +- proxy/src/serverless/conn_pool.rs | 6 +- proxy/src/stream.rs | 2 +- proxy/src/url.rs | 2 +- proxy/src/waiters.rs | 2 +- 36 files changed, 246 insertions(+), 279 deletions(-) diff --git a/proxy/src/auth.rs b/proxy/src/auth.rs index 8c44823c98..3b3c571129 100644 --- a/proxy/src/auth.rs +++ b/proxy/src/auth.rs @@ -113,38 +113,36 @@ impl> From for AuthError { impl UserFacingError for AuthError { fn to_string_client(&self) -> String { - use AuthErrorImpl::*; match self.0.as_ref() { - Link(e) => e.to_string_client(), - GetAuthInfo(e) => e.to_string_client(), - Sasl(e) => e.to_string_client(), - AuthFailed(_) => self.to_string(), - BadAuthMethod(_) => self.to_string(), - MalformedPassword(_) => self.to_string(), - MissingEndpointName => self.to_string(), - Io(_) => "Internal error".to_string(), - IpAddressNotAllowed(_) => self.to_string(), - TooManyConnections => self.to_string(), - UserTimeout(_) => self.to_string(), + AuthErrorImpl::Link(e) => e.to_string_client(), + AuthErrorImpl::GetAuthInfo(e) => e.to_string_client(), + AuthErrorImpl::Sasl(e) => e.to_string_client(), + AuthErrorImpl::AuthFailed(_) => self.to_string(), + AuthErrorImpl::BadAuthMethod(_) => self.to_string(), + AuthErrorImpl::MalformedPassword(_) => self.to_string(), + AuthErrorImpl::MissingEndpointName => self.to_string(), + AuthErrorImpl::Io(_) => "Internal error".to_string(), + AuthErrorImpl::IpAddressNotAllowed(_) => self.to_string(), + AuthErrorImpl::TooManyConnections => self.to_string(), + AuthErrorImpl::UserTimeout(_) => self.to_string(), } } } impl ReportableError for AuthError { fn get_error_kind(&self) -> crate::error::ErrorKind { - use AuthErrorImpl::*; match self.0.as_ref() { - Link(e) => e.get_error_kind(), - GetAuthInfo(e) => e.get_error_kind(), - Sasl(e) => e.get_error_kind(), - AuthFailed(_) => crate::error::ErrorKind::User, - BadAuthMethod(_) => crate::error::ErrorKind::User, - MalformedPassword(_) => crate::error::ErrorKind::User, - MissingEndpointName => crate::error::ErrorKind::User, - Io(_) => crate::error::ErrorKind::ClientDisconnect, - IpAddressNotAllowed(_) => crate::error::ErrorKind::User, - TooManyConnections => crate::error::ErrorKind::RateLimit, - UserTimeout(_) => crate::error::ErrorKind::User, + AuthErrorImpl::Link(e) => e.get_error_kind(), + AuthErrorImpl::GetAuthInfo(e) => e.get_error_kind(), + AuthErrorImpl::Sasl(e) => e.get_error_kind(), + AuthErrorImpl::AuthFailed(_) => crate::error::ErrorKind::User, + AuthErrorImpl::BadAuthMethod(_) => crate::error::ErrorKind::User, + AuthErrorImpl::MalformedPassword(_) => crate::error::ErrorKind::User, + AuthErrorImpl::MissingEndpointName => crate::error::ErrorKind::User, + AuthErrorImpl::Io(_) => crate::error::ErrorKind::ClientDisconnect, + AuthErrorImpl::IpAddressNotAllowed(_) => crate::error::ErrorKind::User, + AuthErrorImpl::TooManyConnections => crate::error::ErrorKind::RateLimit, + AuthErrorImpl::UserTimeout(_) => crate::error::ErrorKind::User, } } } diff --git a/proxy/src/auth/backend.rs b/proxy/src/auth/backend.rs index c6a0b2af5a..7592d076ec 100644 --- a/proxy/src/auth/backend.rs +++ b/proxy/src/auth/backend.rs @@ -80,9 +80,8 @@ pub trait TestBackend: Send + Sync + 'static { impl std::fmt::Display for BackendType<'_, (), ()> { fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - use BackendType::*; match self { - Console(api, _) => match &**api { + Self::Console(api, _) => match &**api { ConsoleBackend::Console(endpoint) => { fmt.debug_tuple("Console").field(&endpoint.url()).finish() } @@ -93,7 +92,7 @@ impl std::fmt::Display for BackendType<'_, (), ()> { #[cfg(test)] ConsoleBackend::Test(_) => fmt.debug_tuple("Test").finish(), }, - Link(url, _) => fmt.debug_tuple("Link").field(&url.as_str()).finish(), + Self::Link(url, _) => fmt.debug_tuple("Link").field(&url.as_str()).finish(), } } } @@ -102,10 +101,9 @@ impl BackendType<'_, T, D> { /// Very similar to [`std::option::Option::as_ref`]. /// This helps us pass structured config to async tasks. pub fn as_ref(&self) -> BackendType<'_, &T, &D> { - use BackendType::*; match self { - Console(c, x) => Console(MaybeOwned::Borrowed(c), x), - Link(c, x) => Link(MaybeOwned::Borrowed(c), x), + Self::Console(c, x) => BackendType::Console(MaybeOwned::Borrowed(c), x), + Self::Link(c, x) => BackendType::Link(MaybeOwned::Borrowed(c), x), } } } @@ -115,10 +113,9 @@ impl<'a, T, D> BackendType<'a, T, D> { /// Maps [`BackendType`] to [`BackendType`] by applying /// a function to a contained value. pub fn map(self, f: impl FnOnce(T) -> R) -> BackendType<'a, R, D> { - use BackendType::*; match self { - Console(c, x) => Console(c, f(x)), - Link(c, x) => Link(c, x), + Self::Console(c, x) => BackendType::Console(c, f(x)), + Self::Link(c, x) => BackendType::Link(c, x), } } } @@ -126,10 +123,9 @@ impl<'a, T, D, E> BackendType<'a, Result, D> { /// Very similar to [`std::option::Option::transpose`]. /// This is most useful for error handling. pub fn transpose(self) -> Result, E> { - use BackendType::*; match self { - Console(c, x) => x.map(|x| Console(c, x)), - Link(c, x) => Ok(Link(c, x)), + Self::Console(c, x) => x.map(|x| BackendType::Console(c, x)), + Self::Link(c, x) => Ok(BackendType::Link(c, x)), } } } @@ -293,7 +289,9 @@ async fn auth_quirks( ctx.set_endpoint_id(res.info.endpoint.clone()); let password = match res.keys { ComputeCredentialKeys::Password(p) => p, - _ => unreachable!("password hack should return a password"), + ComputeCredentialKeys::AuthKeys(_) => { + unreachable!("password hack should return a password") + } }; (res.info, Some(password)) } @@ -400,21 +398,17 @@ async fn authenticate_with_secret( impl<'a> BackendType<'a, ComputeUserInfoMaybeEndpoint, &()> { /// Get compute endpoint name from the credentials. pub fn get_endpoint(&self) -> Option { - use BackendType::*; - match self { - Console(_, user_info) => user_info.endpoint_id.clone(), - Link(_, _) => Some("link".into()), + Self::Console(_, user_info) => user_info.endpoint_id.clone(), + Self::Link(_, _) => Some("link".into()), } } /// Get username from the credentials. pub fn get_user(&self) -> &str { - use BackendType::*; - match self { - Console(_, user_info) => &user_info.user, - Link(_, _) => "link", + Self::Console(_, user_info) => &user_info.user, + Self::Link(_, _) => "link", } } @@ -428,10 +422,8 @@ impl<'a> BackendType<'a, ComputeUserInfoMaybeEndpoint, &()> { config: &'static AuthenticationConfig, endpoint_rate_limiter: Arc, ) -> auth::Result> { - use BackendType::*; - let res = match self { - Console(api, user_info) => { + Self::Console(api, user_info) => { info!( user = &*user_info.user, project = user_info.endpoint(), @@ -451,7 +443,7 @@ impl<'a> BackendType<'a, ComputeUserInfoMaybeEndpoint, &()> { BackendType::Console(api, credentials) } // NOTE: this auth backend doesn't use client credentials. - Link(url, _) => { + Self::Link(url, _) => { info!("performing link authentication"); let info = link::authenticate(ctx, &url, client).await?; @@ -470,10 +462,9 @@ impl BackendType<'_, ComputeUserInfo, &()> { &self, ctx: &RequestMonitoring, ) -> Result { - use BackendType::*; match self { - Console(api, user_info) => api.get_role_secret(ctx, user_info).await, - Link(_, _) => Ok(Cached::new_uncached(None)), + Self::Console(api, user_info) => api.get_role_secret(ctx, user_info).await, + Self::Link(_, _) => Ok(Cached::new_uncached(None)), } } @@ -481,10 +472,9 @@ impl BackendType<'_, ComputeUserInfo, &()> { &self, ctx: &RequestMonitoring, ) -> Result<(CachedAllowedIps, Option), GetAuthInfoError> { - use BackendType::*; match self { - Console(api, user_info) => api.get_allowed_ips_and_secret(ctx, user_info).await, - Link(_, _) => Ok((Cached::new_uncached(Arc::new(vec![])), None)), + Self::Console(api, user_info) => api.get_allowed_ips_and_secret(ctx, user_info).await, + Self::Link(_, _) => Ok((Cached::new_uncached(Arc::new(vec![])), None)), } } } @@ -495,18 +485,16 @@ impl ComputeConnectBackend for BackendType<'_, ComputeCredentials, NodeInfo> { &self, ctx: &RequestMonitoring, ) -> Result { - use BackendType::*; - match self { - Console(api, creds) => api.wake_compute(ctx, &creds.info).await, - Link(_, info) => Ok(Cached::new_uncached(info.clone())), + Self::Console(api, creds) => api.wake_compute(ctx, &creds.info).await, + Self::Link(_, info) => Ok(Cached::new_uncached(info.clone())), } } fn get_keys(&self) -> Option<&ComputeCredentialKeys> { match self { - BackendType::Console(_, creds) => Some(&creds.keys), - BackendType::Link(_, _) => None, + Self::Console(_, creds) => Some(&creds.keys), + Self::Link(_, _) => None, } } } @@ -517,18 +505,16 @@ impl ComputeConnectBackend for BackendType<'_, ComputeCredentials, &()> { &self, ctx: &RequestMonitoring, ) -> Result { - use BackendType::*; - match self { - Console(api, creds) => api.wake_compute(ctx, &creds.info).await, - Link(_, _) => unreachable!("link auth flow doesn't support waking the compute"), + Self::Console(api, creds) => api.wake_compute(ctx, &creds.info).await, + Self::Link(_, _) => unreachable!("link auth flow doesn't support waking the compute"), } } fn get_keys(&self) -> Option<&ComputeCredentialKeys> { match self { - BackendType::Console(_, creds) => Some(&creds.keys), - BackendType::Link(_, _) => None, + Self::Console(_, creds) => Some(&creds.keys), + Self::Link(_, _) => None, } } } diff --git a/proxy/src/auth/backend/jwt.rs b/proxy/src/auth/backend/jwt.rs index 0c2ca8fb97..e021a7e23f 100644 --- a/proxy/src/auth/backend/jwt.rs +++ b/proxy/src/auth/backend/jwt.rs @@ -195,7 +195,7 @@ impl JwkCacheEntryLock { let header = base64::decode_config(header, base64::URL_SAFE_NO_PAD) .context("Provided authentication token is not a valid JWT encoding")?; - let header = serde_json::from_slice::(&header) + let header = serde_json::from_slice::>(&header) .context("Provided authentication token is not a valid JWT encoding")?; let sig = base64::decode_config(signature, base64::URL_SAFE_NO_PAD) @@ -340,7 +340,7 @@ impl JwkRenewalPermit<'_> { } } - async fn acquire_permit(from: &Arc) -> JwkRenewalPermit { + async fn acquire_permit(from: &Arc) -> JwkRenewalPermit<'_> { match from.lookup.acquire().await { Ok(permit) => { permit.forget(); @@ -352,7 +352,7 @@ impl JwkRenewalPermit<'_> { } } - fn try_acquire_permit(from: &Arc) -> Option { + fn try_acquire_permit(from: &Arc) -> Option> { match from.lookup.try_acquire() { Ok(permit) => { permit.forget(); diff --git a/proxy/src/auth/credentials.rs b/proxy/src/auth/credentials.rs index 8f4a392131..849e7d65e8 100644 --- a/proxy/src/auth/credentials.rs +++ b/proxy/src/auth/credentials.rs @@ -89,10 +89,12 @@ impl ComputeUserInfoMaybeEndpoint { sni: Option<&str>, common_names: Option<&HashSet>, ) -> Result { - use ComputeUserInfoParseError::*; - // Some parameters are stored in the startup message. - let get_param = |key| params.get(key).ok_or(MissingKey(key)); + let get_param = |key| { + params + .get(key) + .ok_or(ComputeUserInfoParseError::MissingKey(key)) + }; let user: RoleName = get_param("user")?.into(); // Project name might be passed via PG's command-line options. @@ -122,11 +124,14 @@ impl ComputeUserInfoMaybeEndpoint { let endpoint = match (endpoint_option, endpoint_from_domain) { // Invariant: if we have both project name variants, they should match. (Some(option), Some(domain)) if option != domain => { - Some(Err(InconsistentProjectNames { domain, option })) + Some(Err(ComputeUserInfoParseError::InconsistentProjectNames { + domain, + option, + })) } // Invariant: project name may not contain certain characters. (a, b) => a.or(b).map(|name| match project_name_valid(name.as_ref()) { - false => Err(MalformedProjectName(name)), + false => Err(ComputeUserInfoParseError::MalformedProjectName(name)), true => Ok(name), }), } @@ -186,7 +191,7 @@ impl<'de> serde::de::Deserialize<'de> for IpPattern { impl<'de> serde::de::Visitor<'de> for StrVisitor { type Value = IpPattern; - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(formatter, "comma separated list with ip address, ip address range, or ip address subnet mask") } diff --git a/proxy/src/cache/common.rs b/proxy/src/cache/common.rs index 4e393fddb2..82c78e3eb2 100644 --- a/proxy/src/cache/common.rs +++ b/proxy/src/cache/common.rs @@ -24,7 +24,7 @@ impl Cache for &C { type LookupInfo = C::LookupInfo; fn invalidate(&self, info: &Self::LookupInfo) { - C::invalidate(self, info) + C::invalidate(self, info); } } diff --git a/proxy/src/cache/timed_lru.rs b/proxy/src/cache/timed_lru.rs index c5c4f6a1ed..07fad56643 100644 --- a/proxy/src/cache/timed_lru.rs +++ b/proxy/src/cache/timed_lru.rs @@ -58,7 +58,7 @@ impl Cache for TimedLru { type LookupInfo = LookupInfo; fn invalidate(&self, info: &Self::LookupInfo) { - self.invalidate_raw(info) + self.invalidate_raw(info); } } diff --git a/proxy/src/compute.rs b/proxy/src/compute.rs index 18c82fe379..c071a59d58 100644 --- a/proxy/src/compute.rs +++ b/proxy/src/compute.rs @@ -44,11 +44,10 @@ pub enum ConnectionError { impl UserFacingError for ConnectionError { fn to_string_client(&self) -> String { - use ConnectionError::*; match self { // This helps us drop irrelevant library-specific prefixes. // TODO: propagate severity level and other parameters. - Postgres(err) => match err.as_db_error() { + ConnectionError::Postgres(err) => match err.as_db_error() { Some(err) => { let msg = err.message(); @@ -62,8 +61,8 @@ impl UserFacingError for ConnectionError { } None => err.to_string(), }, - WakeComputeError(err) => err.to_string_client(), - TooManyConnectionAttempts(_) => { + ConnectionError::WakeComputeError(err) => err.to_string_client(), + ConnectionError::TooManyConnectionAttempts(_) => { "Failed to acquire permit to connect to the database. Too many database connection attempts are currently ongoing.".to_owned() } _ => COULD_NOT_CONNECT.to_owned(), @@ -366,16 +365,16 @@ static TLS_ROOTS: OnceCell> = OnceCell::new(); struct AcceptEverythingVerifier; impl ServerCertVerifier for AcceptEverythingVerifier { fn supported_verify_schemes(&self) -> Vec { - use rustls::SignatureScheme::*; + use rustls::SignatureScheme; // The schemes for which `SignatureScheme::supported_in_tls13` returns true. vec![ - ECDSA_NISTP521_SHA512, - ECDSA_NISTP384_SHA384, - ECDSA_NISTP256_SHA256, - RSA_PSS_SHA512, - RSA_PSS_SHA384, - RSA_PSS_SHA256, - ED25519, + SignatureScheme::ECDSA_NISTP521_SHA512, + SignatureScheme::ECDSA_NISTP384_SHA384, + SignatureScheme::ECDSA_NISTP256_SHA256, + SignatureScheme::RSA_PSS_SHA512, + SignatureScheme::RSA_PSS_SHA384, + SignatureScheme::RSA_PSS_SHA256, + SignatureScheme::ED25519, ] } fn verify_server_cert( diff --git a/proxy/src/config.rs b/proxy/src/config.rs index 1412095505..36d04924f2 100644 --- a/proxy/src/config.rs +++ b/proxy/src/config.rs @@ -155,7 +155,7 @@ pub enum TlsServerEndPoint { } impl TlsServerEndPoint { - pub fn new(cert: &CertificateDer) -> anyhow::Result { + pub fn new(cert: &CertificateDer<'_>) -> anyhow::Result { let sha256_oids = [ // I'm explicitly not adding MD5 or SHA1 here... They're bad. oid_registry::OID_SIG_ECDSA_WITH_SHA256, @@ -278,7 +278,7 @@ impl CertResolver { impl rustls::server::ResolvesServerCert for CertResolver { fn resolve( &self, - client_hello: rustls::server::ClientHello, + client_hello: rustls::server::ClientHello<'_>, ) -> Option> { self.resolve(client_hello.server_name()).map(|x| x.0) } @@ -559,7 +559,7 @@ impl RetryConfig { match key { "num_retries" => num_retries = Some(value.parse()?), "base_retry_wait_duration" => { - base_retry_wait_duration = Some(humantime::parse_duration(value)?) + base_retry_wait_duration = Some(humantime::parse_duration(value)?); } "retry_wait_exponent_base" => retry_wait_exponent_base = Some(value.parse()?), unknown => bail!("unknown key: {unknown}"), diff --git a/proxy/src/console/messages.rs b/proxy/src/console/messages.rs index 9abf24ab7f..ac66e116d0 100644 --- a/proxy/src/console/messages.rs +++ b/proxy/src/console/messages.rs @@ -22,16 +22,15 @@ impl ConsoleError { self.status .as_ref() .and_then(|s| s.details.error_info.as_ref()) - .map(|e| e.reason) - .unwrap_or(Reason::Unknown) + .map_or(Reason::Unknown, |e| e.reason) } + pub fn get_user_facing_message(&self) -> String { use super::provider::errors::REQUEST_FAILED; self.status .as_ref() .and_then(|s| s.details.user_facing_message.as_ref()) - .map(|m| m.message.clone().into()) - .unwrap_or_else(|| { + .map_or_else(|| { // Ask @neondatabase/control-plane for review before adding more. match self.http_status_code { http::StatusCode::NOT_FOUND => { @@ -48,19 +47,18 @@ impl ConsoleError { } _ => REQUEST_FAILED.to_owned(), } - }) + }, |m| m.message.clone().into()) } } impl Display for ConsoleError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let msg = self + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let msg: &str = self .status .as_ref() .and_then(|s| s.details.user_facing_message.as_ref()) - .map(|m| m.message.as_ref()) - .unwrap_or_else(|| &self.error); - write!(f, "{}", msg) + .map_or_else(|| self.error.as_ref(), |m| m.message.as_ref()); + write!(f, "{msg}") } } @@ -286,7 +284,7 @@ pub struct DatabaseInfo { // Manually implement debug to omit sensitive info. impl fmt::Debug for DatabaseInfo { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("DatabaseInfo") .field("host", &self.host) .field("port", &self.port) @@ -373,7 +371,7 @@ mod tests { } } }); - let _: KickSession = serde_json::from_str(&json.to_string())?; + let _: KickSession<'_> = serde_json::from_str(&json.to_string())?; Ok(()) } diff --git a/proxy/src/console/mgmt.rs b/proxy/src/console/mgmt.rs index befe7d7510..82d5033aab 100644 --- a/proxy/src/console/mgmt.rs +++ b/proxy/src/console/mgmt.rs @@ -93,7 +93,8 @@ impl postgres_backend::Handler for MgmtHandler { } fn try_process_query(pgb: &mut PostgresBackendTCP, query: &str) -> Result<(), QueryError> { - let resp: KickSession = serde_json::from_str(query).context("Failed to parse query as json")?; + let resp: KickSession<'_> = + serde_json::from_str(query).context("Failed to parse query as json")?; let span = info_span!("event", session_id = resp.session_id); let _enter = span.enter(); diff --git a/proxy/src/console/provider.rs b/proxy/src/console/provider.rs index 15fc0134b3..cc2ee10062 100644 --- a/proxy/src/console/provider.rs +++ b/proxy/src/console/provider.rs @@ -26,7 +26,7 @@ use tracing::info; pub mod errors { use crate::{ console::messages::{self, ConsoleError, Reason}, - error::{io_error, ReportableError, UserFacingError}, + error::{io_error, ErrorKind, ReportableError, UserFacingError}, proxy::retry::CouldRetry, }; use thiserror::Error; @@ -51,21 +51,19 @@ pub mod errors { impl ApiError { /// Returns HTTP status code if it's the reason for failure. pub fn get_reason(&self) -> messages::Reason { - use ApiError::*; match self { - Console(e) => e.get_reason(), - _ => messages::Reason::Unknown, + ApiError::Console(e) => e.get_reason(), + ApiError::Transport(_) => messages::Reason::Unknown, } } } impl UserFacingError for ApiError { fn to_string_client(&self) -> String { - use ApiError::*; match self { // To minimize risks, only select errors are forwarded to users. - Console(c) => c.get_user_facing_message(), - _ => REQUEST_FAILED.to_owned(), + ApiError::Console(c) => c.get_user_facing_message(), + ApiError::Transport(_) => REQUEST_FAILED.to_owned(), } } } @@ -73,57 +71,53 @@ pub mod errors { impl ReportableError for ApiError { fn get_error_kind(&self) -> crate::error::ErrorKind { match self { - ApiError::Console(e) => { - use crate::error::ErrorKind::*; - match e.get_reason() { - Reason::RoleProtected => User, - Reason::ResourceNotFound => User, - Reason::ProjectNotFound => User, - Reason::EndpointNotFound => User, - Reason::BranchNotFound => User, - Reason::RateLimitExceeded => ServiceRateLimit, - Reason::NonDefaultBranchComputeTimeExceeded => User, - Reason::ActiveTimeQuotaExceeded => User, - Reason::ComputeTimeQuotaExceeded => User, - Reason::WrittenDataQuotaExceeded => User, - Reason::DataTransferQuotaExceeded => User, - Reason::LogicalSizeQuotaExceeded => User, - Reason::ConcurrencyLimitReached => ControlPlane, - Reason::LockAlreadyTaken => ControlPlane, - Reason::RunningOperations => ControlPlane, - Reason::Unknown => match &e { - ConsoleError { - http_status_code: - http::StatusCode::NOT_FOUND | http::StatusCode::NOT_ACCEPTABLE, - .. - } => crate::error::ErrorKind::User, - ConsoleError { - http_status_code: http::StatusCode::UNPROCESSABLE_ENTITY, - error, - .. - } if error.contains( - "compute time quota of non-primary branches is exceeded", - ) => - { - crate::error::ErrorKind::User - } - ConsoleError { - http_status_code: http::StatusCode::LOCKED, - error, - .. - } if error.contains("quota exceeded") - || error.contains("the limit for current plan reached") => - { - crate::error::ErrorKind::User - } - ConsoleError { - http_status_code: http::StatusCode::TOO_MANY_REQUESTS, - .. - } => crate::error::ErrorKind::ServiceRateLimit, - ConsoleError { .. } => crate::error::ErrorKind::ControlPlane, - }, - } - } + ApiError::Console(e) => match e.get_reason() { + Reason::RoleProtected => ErrorKind::User, + Reason::ResourceNotFound => ErrorKind::User, + Reason::ProjectNotFound => ErrorKind::User, + Reason::EndpointNotFound => ErrorKind::User, + Reason::BranchNotFound => ErrorKind::User, + Reason::RateLimitExceeded => ErrorKind::ServiceRateLimit, + Reason::NonDefaultBranchComputeTimeExceeded => ErrorKind::User, + Reason::ActiveTimeQuotaExceeded => ErrorKind::User, + Reason::ComputeTimeQuotaExceeded => ErrorKind::User, + Reason::WrittenDataQuotaExceeded => ErrorKind::User, + Reason::DataTransferQuotaExceeded => ErrorKind::User, + Reason::LogicalSizeQuotaExceeded => ErrorKind::User, + Reason::ConcurrencyLimitReached => ErrorKind::ControlPlane, + Reason::LockAlreadyTaken => ErrorKind::ControlPlane, + Reason::RunningOperations => ErrorKind::ControlPlane, + Reason::Unknown => match &e { + ConsoleError { + http_status_code: + http::StatusCode::NOT_FOUND | http::StatusCode::NOT_ACCEPTABLE, + .. + } => crate::error::ErrorKind::User, + ConsoleError { + http_status_code: http::StatusCode::UNPROCESSABLE_ENTITY, + error, + .. + } if error + .contains("compute time quota of non-primary branches is exceeded") => + { + crate::error::ErrorKind::User + } + ConsoleError { + http_status_code: http::StatusCode::LOCKED, + error, + .. + } if error.contains("quota exceeded") + || error.contains("the limit for current plan reached") => + { + crate::error::ErrorKind::User + } + ConsoleError { + http_status_code: http::StatusCode::TOO_MANY_REQUESTS, + .. + } => crate::error::ErrorKind::ServiceRateLimit, + ConsoleError { .. } => crate::error::ErrorKind::ControlPlane, + }, + }, ApiError::Transport(_) => crate::error::ErrorKind::ControlPlane, } } @@ -170,12 +164,11 @@ pub mod errors { impl UserFacingError for GetAuthInfoError { fn to_string_client(&self) -> String { - use GetAuthInfoError::*; match self { // We absolutely should not leak any secrets! - BadSecret => REQUEST_FAILED.to_owned(), + Self::BadSecret => REQUEST_FAILED.to_owned(), // However, API might return a meaningful error. - ApiError(e) => e.to_string_client(), + Self::ApiError(e) => e.to_string_client(), } } } @@ -183,8 +176,8 @@ pub mod errors { impl ReportableError for GetAuthInfoError { fn get_error_kind(&self) -> crate::error::ErrorKind { match self { - GetAuthInfoError::BadSecret => crate::error::ErrorKind::ControlPlane, - GetAuthInfoError::ApiError(_) => crate::error::ErrorKind::ControlPlane, + Self::BadSecret => crate::error::ErrorKind::ControlPlane, + Self::ApiError(_) => crate::error::ErrorKind::ControlPlane, } } } @@ -213,17 +206,16 @@ pub mod errors { impl UserFacingError for WakeComputeError { fn to_string_client(&self) -> String { - use WakeComputeError::*; match self { // We shouldn't show user the address even if it's broken. // Besides, user is unlikely to care about this detail. - BadComputeAddress(_) => REQUEST_FAILED.to_owned(), + Self::BadComputeAddress(_) => REQUEST_FAILED.to_owned(), // However, API might return a meaningful error. - ApiError(e) => e.to_string_client(), + Self::ApiError(e) => e.to_string_client(), - TooManyConnections => self.to_string(), + Self::TooManyConnections => self.to_string(), - TooManyConnectionAttempts(_) => { + Self::TooManyConnectionAttempts(_) => { "Failed to acquire permit to connect to the database. Too many database connection attempts are currently ongoing.".to_owned() } } @@ -233,10 +225,10 @@ pub mod errors { impl ReportableError for WakeComputeError { fn get_error_kind(&self) -> crate::error::ErrorKind { match self { - WakeComputeError::BadComputeAddress(_) => crate::error::ErrorKind::ControlPlane, - WakeComputeError::ApiError(e) => e.get_error_kind(), - WakeComputeError::TooManyConnections => crate::error::ErrorKind::RateLimit, - WakeComputeError::TooManyConnectionAttempts(e) => e.get_error_kind(), + Self::BadComputeAddress(_) => crate::error::ErrorKind::ControlPlane, + Self::ApiError(e) => e.get_error_kind(), + Self::TooManyConnections => crate::error::ErrorKind::RateLimit, + Self::TooManyConnectionAttempts(e) => e.get_error_kind(), } } } @@ -244,10 +236,10 @@ pub mod errors { impl CouldRetry for WakeComputeError { fn could_retry(&self) -> bool { match self { - WakeComputeError::BadComputeAddress(_) => false, - WakeComputeError::ApiError(e) => e.could_retry(), - WakeComputeError::TooManyConnections => false, - WakeComputeError::TooManyConnectionAttempts(_) => false, + Self::BadComputeAddress(_) => false, + Self::ApiError(e) => e.could_retry(), + Self::TooManyConnections => false, + Self::TooManyConnectionAttempts(_) => false, } } } @@ -366,13 +358,14 @@ impl Api for ConsoleBackend { ctx: &RequestMonitoring, user_info: &ComputeUserInfo, ) -> Result { - use ConsoleBackend::*; match self { - Console(api) => api.get_role_secret(ctx, user_info).await, + Self::Console(api) => api.get_role_secret(ctx, user_info).await, #[cfg(any(test, feature = "testing"))] - Postgres(api) => api.get_role_secret(ctx, user_info).await, + Self::Postgres(api) => api.get_role_secret(ctx, user_info).await, #[cfg(test)] - Test(_) => unreachable!("this function should never be called in the test backend"), + Self::Test(_) => { + unreachable!("this function should never be called in the test backend") + } } } @@ -381,13 +374,12 @@ impl Api for ConsoleBackend { ctx: &RequestMonitoring, user_info: &ComputeUserInfo, ) -> Result<(CachedAllowedIps, Option), errors::GetAuthInfoError> { - use ConsoleBackend::*; match self { - Console(api) => api.get_allowed_ips_and_secret(ctx, user_info).await, + Self::Console(api) => api.get_allowed_ips_and_secret(ctx, user_info).await, #[cfg(any(test, feature = "testing"))] - Postgres(api) => api.get_allowed_ips_and_secret(ctx, user_info).await, + Self::Postgres(api) => api.get_allowed_ips_and_secret(ctx, user_info).await, #[cfg(test)] - Test(api) => api.get_allowed_ips_and_secret(), + Self::Test(api) => api.get_allowed_ips_and_secret(), } } @@ -396,14 +388,12 @@ impl Api for ConsoleBackend { ctx: &RequestMonitoring, user_info: &ComputeUserInfo, ) -> Result { - use ConsoleBackend::*; - match self { - Console(api) => api.wake_compute(ctx, user_info).await, + Self::Console(api) => api.wake_compute(ctx, user_info).await, #[cfg(any(test, feature = "testing"))] - Postgres(api) => api.wake_compute(ctx, user_info).await, + Self::Postgres(api) => api.wake_compute(ctx, user_info).await, #[cfg(test)] - Test(api) => api.wake_compute(), + Self::Test(api) => api.wake_compute(), } } } @@ -549,7 +539,7 @@ impl WakeComputePermit { !self.permit.is_disabled() } pub fn release(self, outcome: Outcome) { - self.permit.release(outcome) + self.permit.release(outcome); } pub fn release_result(self, res: Result) -> Result { match res { diff --git a/proxy/src/context.rs b/proxy/src/context.rs index e925f67233..cafbdedc15 100644 --- a/proxy/src/context.rs +++ b/proxy/src/context.rs @@ -166,7 +166,7 @@ impl RequestMonitoring { pub fn set_project(&self, x: MetricsAuxInfo) { let mut this = self.0.try_lock().expect("should not deadlock"); if this.endpoint_id.is_none() { - this.set_endpoint_id(x.endpoint_id.as_str().into()) + this.set_endpoint_id(x.endpoint_id.as_str().into()); } this.branch = Some(x.branch_id); this.project = Some(x.project_id); @@ -260,7 +260,7 @@ impl RequestMonitoring { .cold_start_info } - pub fn latency_timer_pause(&self, waiting_for: Waiting) -> LatencyTimerPause { + pub fn latency_timer_pause(&self, waiting_for: Waiting) -> LatencyTimerPause<'_> { LatencyTimerPause { ctx: self, start: tokio::time::Instant::now(), @@ -273,7 +273,7 @@ impl RequestMonitoring { .try_lock() .expect("should not deadlock") .latency_timer - .success() + .success(); } } @@ -328,7 +328,7 @@ impl RequestMonitoringInner { fn has_private_peer_addr(&self) -> bool { match self.peer_addr { IpAddr::V4(ip) => ip.is_private(), - _ => false, + IpAddr::V6(_) => false, } } diff --git a/proxy/src/context/parquet.rs b/proxy/src/context/parquet.rs index bb02a476fc..e5962b35fa 100644 --- a/proxy/src/context/parquet.rs +++ b/proxy/src/context/parquet.rs @@ -736,7 +736,7 @@ mod tests { while let Some(r) = s.next().await { tx.send(r).unwrap(); } - time::sleep(time::Duration::from_secs(70)).await + time::sleep(time::Duration::from_secs(70)).await; } }); diff --git a/proxy/src/intern.rs b/proxy/src/intern.rs index e38135dd22..d418caa511 100644 --- a/proxy/src/intern.rs +++ b/proxy/src/intern.rs @@ -56,7 +56,7 @@ impl<'de, Id: InternId> serde::de::Deserialize<'de> for InternedString { impl<'de, Id: InternId> serde::de::Visitor<'de> for Visitor { type Value = InternedString; - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter.write_str("a string") } diff --git a/proxy/src/metrics.rs b/proxy/src/metrics.rs index 0167553e30..ccef88231b 100644 --- a/proxy/src/metrics.rs +++ b/proxy/src/metrics.rs @@ -252,7 +252,7 @@ impl Drop for HttpEndpointPoolsGuard<'_> { } impl HttpEndpointPools { - pub fn guard(&self) -> HttpEndpointPoolsGuard { + pub fn guard(&self) -> HttpEndpointPoolsGuard<'_> { self.http_pool_endpoints_registered_total.inc(); HttpEndpointPoolsGuard { dec: &self.http_pool_endpoints_unregistered_total, diff --git a/proxy/src/proxy/copy_bidirectional.rs b/proxy/src/proxy/copy_bidirectional.rs index 3c45fff969..048523f69c 100644 --- a/proxy/src/proxy/copy_bidirectional.rs +++ b/proxy/src/proxy/copy_bidirectional.rs @@ -184,7 +184,7 @@ impl CopyBuffer { } Poll::Pending } - res => res.map_err(ErrorDirection::Write), + res @ Poll::Ready(_) => res.map_err(ErrorDirection::Write), } } diff --git a/proxy/src/proxy/handshake.rs b/proxy/src/proxy/handshake.rs index c65a5558d9..27a72f8072 100644 --- a/proxy/src/proxy/handshake.rs +++ b/proxy/src/proxy/handshake.rs @@ -82,9 +82,8 @@ pub async fn handshake( let mut stream = PqStream::new(Stream::from_raw(stream)); loop { let msg = stream.read_startup_packet().await?; - use FeStartupPacket::*; match msg { - SslRequest { direct } => match stream.get_ref() { + FeStartupPacket::SslRequest { direct } => match stream.get_ref() { Stream::Raw { .. } if !tried_ssl => { tried_ssl = true; @@ -139,7 +138,7 @@ pub async fn handshake( let tls_stream = accept.await.inspect_err(|_| { if record_handshake_error { - Metrics::get().proxy.tls_handshake_failures.inc() + Metrics::get().proxy.tls_handshake_failures.inc(); } })?; @@ -182,7 +181,7 @@ pub async fn handshake( } _ => return Err(HandshakeError::ProtocolViolation), }, - GssEncRequest => match stream.get_ref() { + FeStartupPacket::GssEncRequest => match stream.get_ref() { Stream::Raw { .. } if !tried_gss => { tried_gss = true; @@ -191,7 +190,7 @@ pub async fn handshake( } _ => return Err(HandshakeError::ProtocolViolation), }, - StartupMessage { params, version } + FeStartupPacket::StartupMessage { params, version } if PG_PROTOCOL_EARLIEST <= version && version <= PG_PROTOCOL_LATEST => { // Check that the config has been consumed during upgrade @@ -211,7 +210,7 @@ pub async fn handshake( break Ok(HandshakeData::Startup(stream, params)); } // downgrade protocol version - StartupMessage { params, version } + FeStartupPacket::StartupMessage { params, version } if version.major() == 3 && version > PG_PROTOCOL_LATEST => { warn!(?version, "unsupported minor version"); @@ -241,7 +240,7 @@ pub async fn handshake( ); break Ok(HandshakeData::Startup(stream, params)); } - StartupMessage { version, .. } => { + FeStartupPacket::StartupMessage { version, .. } => { warn!( ?version, session_type = "normal", @@ -249,7 +248,7 @@ pub async fn handshake( ); return Err(HandshakeError::ProtocolViolation); } - CancelRequest(cancel_key_data) => { + FeStartupPacket::CancelRequest(cancel_key_data) => { info!(session_type = "cancellation", "successful handshake"); break Ok(HandshakeData::Cancel(cancel_key_data)); } diff --git a/proxy/src/proxy/tests/mitm.rs b/proxy/src/proxy/tests/mitm.rs index c8ec2b2db6..2d752b9183 100644 --- a/proxy/src/proxy/tests/mitm.rs +++ b/proxy/src/proxy/tests/mitm.rs @@ -68,7 +68,7 @@ async fn proxy_mitm( end_client.send(Bytes::from_static(b"R\0\0\0\x17\0\0\0\x0aSCRAM-SHA-256\0\0")).await.unwrap(); continue; } - end_client.send(message).await.unwrap() + end_client.send(message).await.unwrap(); } _ => break, } @@ -88,7 +88,7 @@ async fn proxy_mitm( end_server.send(buf.freeze()).await.unwrap(); continue; } - end_server.send(message).await.unwrap() + end_server.send(message).await.unwrap(); } _ => break, } diff --git a/proxy/src/rate_limiter/limit_algorithm.rs b/proxy/src/rate_limiter/limit_algorithm.rs index 3842ce269e..80a62b2a76 100644 --- a/proxy/src/rate_limiter/limit_algorithm.rs +++ b/proxy/src/rate_limiter/limit_algorithm.rs @@ -237,7 +237,7 @@ impl Token { } pub fn release(mut self, outcome: Outcome) { - self.release_mut(Some(outcome)) + self.release_mut(Some(outcome)); } pub fn release_mut(&mut self, outcome: Option) { @@ -249,7 +249,7 @@ impl Token { impl Drop for Token { fn drop(&mut self) { - self.release_mut(None) + self.release_mut(None); } } diff --git a/proxy/src/rate_limiter/limit_algorithm/aimd.rs b/proxy/src/rate_limiter/limit_algorithm/aimd.rs index b39740bb21..d669492fa6 100644 --- a/proxy/src/rate_limiter/limit_algorithm/aimd.rs +++ b/proxy/src/rate_limiter/limit_algorithm/aimd.rs @@ -25,9 +25,8 @@ pub struct Aimd { impl LimitAlgorithm for Aimd { fn update(&self, old_limit: usize, sample: Sample) -> usize { - use Outcome::*; match sample.outcome { - Success => { + Outcome::Success => { let utilisation = sample.in_flight as f32 / old_limit as f32; if utilisation > self.utilisation { @@ -42,7 +41,7 @@ impl LimitAlgorithm for Aimd { old_limit } } - Overload => { + Outcome::Overload => { let limit = old_limit as f32 * self.dec; // Floor instead of round, so the limit reduces even with small numbers. diff --git a/proxy/src/redis/connection_with_credentials_provider.rs b/proxy/src/redis/connection_with_credentials_provider.rs index b02ce472c0..c78ee166f1 100644 --- a/proxy/src/redis/connection_with_credentials_provider.rs +++ b/proxy/src/redis/connection_with_credentials_provider.rs @@ -98,7 +98,7 @@ impl ConnectionWithCredentialsProvider { info!("Establishing a new connection..."); self.con = None; if let Some(f) = self.refresh_token_task.take() { - f.abort() + f.abort(); } let mut con = self .get_client() diff --git a/proxy/src/redis/notifications.rs b/proxy/src/redis/notifications.rs index efd7437d5d..ad69246443 100644 --- a/proxy/src/redis/notifications.rs +++ b/proxy/src/redis/notifications.rs @@ -108,7 +108,6 @@ impl MessageHandler { } #[tracing::instrument(skip(self, msg), fields(session_id = tracing::field::Empty))] async fn handle_message(&self, msg: redis::Msg) -> anyhow::Result<()> { - use Notification::*; let payload: String = msg.get_payload()?; tracing::debug!(?payload, "received a message payload"); @@ -124,7 +123,7 @@ impl MessageHandler { }; tracing::debug!(?msg, "received a message"); match msg { - Cancel(cancel_session) => { + Notification::Cancel(cancel_session) => { tracing::Span::current().record( "session_id", tracing::field::display(cancel_session.session_id), @@ -153,12 +152,12 @@ impl MessageHandler { } _ => { invalidate_cache(self.cache.clone(), msg.clone()); - if matches!(msg, AllowedIpsUpdate { .. }) { + if matches!(msg, Notification::AllowedIpsUpdate { .. }) { Metrics::get() .proxy .redis_events_count .inc(RedisEventsCount::AllowedIpsUpdate); - } else if matches!(msg, PasswordUpdate { .. }) { + } else if matches!(msg, Notification::PasswordUpdate { .. }) { Metrics::get() .proxy .redis_events_count @@ -180,16 +179,16 @@ impl MessageHandler { } fn invalidate_cache(cache: Arc, msg: Notification) { - use Notification::*; match msg { - AllowedIpsUpdate { allowed_ips_update } => { - cache.invalidate_allowed_ips_for_project(allowed_ips_update.project_id) + Notification::AllowedIpsUpdate { allowed_ips_update } => { + cache.invalidate_allowed_ips_for_project(allowed_ips_update.project_id); } - PasswordUpdate { password_update } => cache.invalidate_role_secret_for_project( - password_update.project_id, - password_update.role_name, - ), - Cancel(_) => unreachable!("cancel message should be handled separately"), + Notification::PasswordUpdate { password_update } => cache + .invalidate_role_secret_for_project( + password_update.project_id, + password_update.role_name, + ), + Notification::Cancel(_) => unreachable!("cancel message should be handled separately"), } } diff --git a/proxy/src/sasl.rs b/proxy/src/sasl.rs index 0811416ca2..60207fc824 100644 --- a/proxy/src/sasl.rs +++ b/proxy/src/sasl.rs @@ -42,10 +42,9 @@ pub enum Error { impl UserFacingError for Error { fn to_string_client(&self) -> String { - use Error::*; match self { - ChannelBindingFailed(m) => m.to_string(), - ChannelBindingBadMethod(m) => format!("unsupported channel binding method {m}"), + Self::ChannelBindingFailed(m) => (*m).to_string(), + Self::ChannelBindingBadMethod(m) => format!("unsupported channel binding method {m}"), _ => "authentication protocol violation".to_string(), } } diff --git a/proxy/src/sasl/channel_binding.rs b/proxy/src/sasl/channel_binding.rs index 13d681de6d..6e2d3057ce 100644 --- a/proxy/src/sasl/channel_binding.rs +++ b/proxy/src/sasl/channel_binding.rs @@ -13,11 +13,10 @@ pub enum ChannelBinding { impl ChannelBinding { pub fn and_then(self, f: impl FnOnce(T) -> Result) -> Result, E> { - use ChannelBinding::*; Ok(match self { - NotSupportedClient => NotSupportedClient, - NotSupportedServer => NotSupportedServer, - Required(x) => Required(f(x)?), + Self::NotSupportedClient => ChannelBinding::NotSupportedClient, + Self::NotSupportedServer => ChannelBinding::NotSupportedServer, + Self::Required(x) => ChannelBinding::Required(f(x)?), }) } } @@ -25,11 +24,10 @@ impl ChannelBinding { impl<'a> ChannelBinding<&'a str> { // NB: FromStr doesn't work with lifetimes pub fn parse(input: &'a str) -> Option { - use ChannelBinding::*; Some(match input { - "n" => NotSupportedClient, - "y" => NotSupportedServer, - other => Required(other.strip_prefix("p=")?), + "n" => Self::NotSupportedClient, + "y" => Self::NotSupportedServer, + other => Self::Required(other.strip_prefix("p=")?), }) } } @@ -40,17 +38,16 @@ impl ChannelBinding { &self, get_cbind_data: impl FnOnce(&T) -> Result<&'a [u8], E>, ) -> Result, E> { - use ChannelBinding::*; Ok(match self { - NotSupportedClient => { + Self::NotSupportedClient => { // base64::encode("n,,") "biws".into() } - NotSupportedServer => { + Self::NotSupportedServer => { // base64::encode("y,,") "eSws".into() } - Required(mode) => { + Self::Required(mode) => { use std::io::Write; let mut cbind_input = vec![]; write!(&mut cbind_input, "p={mode},,",).unwrap(); diff --git a/proxy/src/sasl/messages.rs b/proxy/src/sasl/messages.rs index b9208f6f1f..2b5ae1785d 100644 --- a/proxy/src/sasl/messages.rs +++ b/proxy/src/sasl/messages.rs @@ -42,10 +42,9 @@ pub(super) enum ServerMessage { impl<'a> ServerMessage<&'a str> { pub(super) fn to_reply(&self) -> BeMessage<'a> { - use BeAuthenticationSaslMessage::*; BeMessage::AuthenticationSasl(match self { - ServerMessage::Continue(s) => Continue(s.as_bytes()), - ServerMessage::Final(s) => Final(s.as_bytes()), + ServerMessage::Continue(s) => BeAuthenticationSaslMessage::Continue(s.as_bytes()), + ServerMessage::Final(s) => BeAuthenticationSaslMessage::Final(s.as_bytes()), }) } } diff --git a/proxy/src/scram.rs b/proxy/src/scram.rs index 862facb4e5..145e727a74 100644 --- a/proxy/src/scram.rs +++ b/proxy/src/scram.rs @@ -137,12 +137,12 @@ mod tests { #[tokio::test] async fn round_trip() { - run_round_trip_test("pencil", "pencil").await + run_round_trip_test("pencil", "pencil").await; } #[tokio::test] #[should_panic(expected = "password doesn't match")] async fn failure() { - run_round_trip_test("pencil", "eraser").await + run_round_trip_test("pencil", "eraser").await; } } diff --git a/proxy/src/scram/countmin.rs b/proxy/src/scram/countmin.rs index e8e7ef5c86..944bb3c83e 100644 --- a/proxy/src/scram/countmin.rs +++ b/proxy/src/scram/countmin.rs @@ -98,8 +98,6 @@ mod tests { // q% of counts will be within p of the actual value let mut sketch = CountMinSketch::with_params(p / N as f64, 1.0 - q); - dbg!(sketch.buckets.len()); - // insert a bunch of entries in a random order let mut ids2 = ids.clone(); while !ids2.is_empty() { diff --git a/proxy/src/scram/exchange.rs b/proxy/src/scram/exchange.rs index d0adbc780e..f2494379a5 100644 --- a/proxy/src/scram/exchange.rs +++ b/proxy/src/scram/exchange.rs @@ -210,23 +210,23 @@ impl sasl::Mechanism for Exchange<'_> { type Output = super::ScramKey; fn exchange(mut self, input: &str) -> sasl::Result> { - use {sasl::Step::*, ExchangeState::*}; + use {sasl::Step, ExchangeState}; match &self.state { - Initial(init) => { + ExchangeState::Initial(init) => { match init.transition(self.secret, &self.tls_server_end_point, input)? { - Continue(sent, msg) => { - self.state = SaltSent(sent); - Ok(Continue(self, msg)) + Step::Continue(sent, msg) => { + self.state = ExchangeState::SaltSent(sent); + Ok(Step::Continue(self, msg)) } - Success(x, _) => match x {}, - Failure(msg) => Ok(Failure(msg)), + Step::Success(x, _) => match x {}, + Step::Failure(msg) => Ok(Step::Failure(msg)), } } - SaltSent(sent) => { + ExchangeState::SaltSent(sent) => { match sent.transition(self.secret, &self.tls_server_end_point, input)? { - Success(keys, msg) => Ok(Success(keys, msg)), - Continue(x, _) => match x {}, - Failure(msg) => Ok(Failure(msg)), + Step::Success(keys, msg) => Ok(Step::Success(keys, msg)), + Step::Continue(x, _) => match x {}, + Step::Failure(msg) => Ok(Step::Failure(msg)), } } } diff --git a/proxy/src/scram/messages.rs b/proxy/src/scram/messages.rs index cf677a3334..5ecbbf7004 100644 --- a/proxy/src/scram/messages.rs +++ b/proxy/src/scram/messages.rs @@ -59,7 +59,7 @@ impl<'a> ClientFirstMessage<'a> { // https://github.com/postgres/postgres/blob/f83908798f78c4cafda217ca875602c88ea2ae28/src/backend/libpq/auth-scram.c#L13-L14 if !username.is_empty() { - tracing::warn!(username, "scram username provided, but is not expected") + tracing::warn!(username, "scram username provided, but is not expected"); // TODO(conrad): // return None; } @@ -137,7 +137,7 @@ impl<'a> ClientFinalMessage<'a> { /// Build a response to [`ClientFinalMessage`]. pub fn build_server_final_message( &self, - signature_builder: SignatureBuilder, + signature_builder: SignatureBuilder<'_>, server_key: &ScramKey, ) -> String { let mut buf = String::from("v="); @@ -212,7 +212,7 @@ mod tests { #[test] fn parse_client_first_message_with_invalid_gs2_authz() { - assert!(ClientFirstMessage::parse("n,authzid,n=,r=nonce").is_none()) + assert!(ClientFirstMessage::parse("n,authzid,n=,r=nonce").is_none()); } #[test] diff --git a/proxy/src/scram/pbkdf2.rs b/proxy/src/scram/pbkdf2.rs index a803ba7e1b..f690cc7738 100644 --- a/proxy/src/scram/pbkdf2.rs +++ b/proxy/src/scram/pbkdf2.rs @@ -84,6 +84,6 @@ mod tests { }; let expected = pbkdf2_hmac_array::(pass, salt, 600000); - assert_eq!(hash, expected) + assert_eq!(hash, expected); } } diff --git a/proxy/src/scram/threadpool.rs b/proxy/src/scram/threadpool.rs index 7701b869a3..fa3d3ccca2 100644 --- a/proxy/src/scram/threadpool.rs +++ b/proxy/src/scram/threadpool.rs @@ -270,7 +270,7 @@ fn thread_rt(pool: Arc, worker: Worker, index: usize) { .inc(ThreadPoolWorkerId(index)); // skip for now - worker.push(job) + worker.push(job); } } @@ -316,6 +316,6 @@ mod tests { 10, 114, 73, 188, 140, 222, 196, 156, 214, 184, 79, 157, 119, 242, 16, 31, 53, 242, 178, 43, 95, 8, 225, 182, 122, 40, 219, 21, 89, 147, 64, 140, ]; - assert_eq!(actual, expected) + assert_eq!(actual, expected); } } diff --git a/proxy/src/serverless.rs b/proxy/src/serverless.rs index 115bef7375..5416d63b5b 100644 --- a/proxy/src/serverless.rs +++ b/proxy/src/serverless.rs @@ -120,7 +120,7 @@ pub async fn task_main( tracing::trace!("attempting to cancel a random connection"); if let Some(token) = config.http_config.cancel_set.take() { tracing::debug!("cancelling a random connection"); - token.cancel() + token.cancel(); } } @@ -198,7 +198,7 @@ async fn connection_startup( let peer_addr = peer.unwrap_or(peer_addr).ip(); let has_private_peer_addr = match peer_addr { IpAddr::V4(ip) => ip.is_private(), - _ => false, + IpAddr::V6(_) => false, }; info!(?session_id, %peer_addr, "accepted new TCP connection"); diff --git a/proxy/src/serverless/conn_pool.rs b/proxy/src/serverless/conn_pool.rs index e1dc44dc1c..9ede659cc4 100644 --- a/proxy/src/serverless/conn_pool.rs +++ b/proxy/src/serverless/conn_pool.rs @@ -390,7 +390,7 @@ impl GlobalConnPool { .write() .get_conn_entry(conn_info.db_and_user()) { - client = Some(entry.conn) + client = Some(entry.conn); } let endpoint_pool = Arc::downgrade(&endpoint_pool); @@ -662,13 +662,13 @@ impl Discard<'_, C> { pub fn check_idle(&mut self, status: ReadyForQueryStatus) { let conn_info = &self.conn_info; if status != ReadyForQueryStatus::Idle && std::mem::take(self.pool).strong_count() > 0 { - info!("pool: throwing away connection '{conn_info}' because connection is not idle") + info!("pool: throwing away connection '{conn_info}' because connection is not idle"); } } pub fn discard(&mut self) { let conn_info = &self.conn_info; if std::mem::take(self.pool).strong_count() > 0 { - info!("pool: throwing away connection '{conn_info}' because connection is potentially in a broken state") + info!("pool: throwing away connection '{conn_info}' because connection is potentially in a broken state"); } } } diff --git a/proxy/src/stream.rs b/proxy/src/stream.rs index 690e92ffb1..7809d2e574 100644 --- a/proxy/src/stream.rs +++ b/proxy/src/stream.rs @@ -234,7 +234,7 @@ impl Stream { .await .inspect_err(|_| { if record_handshake_error { - Metrics::get().proxy.tls_handshake_failures.inc() + Metrics::get().proxy.tls_handshake_failures.inc(); } })?), Stream::Tls { .. } => Err(StreamUpgradeError::AlreadyTls), diff --git a/proxy/src/url.rs b/proxy/src/url.rs index 92c64bb8ad..202fe8de1f 100644 --- a/proxy/src/url.rs +++ b/proxy/src/url.rs @@ -12,7 +12,7 @@ impl ApiUrl { } /// See [`url::Url::path_segments_mut`]. - pub fn path_segments_mut(&mut self) -> url::PathSegmentsMut { + pub fn path_segments_mut(&mut self) -> url::PathSegmentsMut<'_> { // We've already verified that it works during construction. self.0.path_segments_mut().expect("bad API url") } diff --git a/proxy/src/waiters.rs b/proxy/src/waiters.rs index 888ad38048..3bd8f4c8ef 100644 --- a/proxy/src/waiters.rs +++ b/proxy/src/waiters.rs @@ -36,7 +36,7 @@ impl Default for Waiters { } impl Waiters { - pub fn register(&self, key: String) -> Result, RegisterError> { + pub fn register(&self, key: String) -> Result, RegisterError> { let (tx, rx) = oneshot::channel(); self.0 From eb7241c798d445cd7bcb52d14fbf6c59f4a54d32 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Mon, 19 Aug 2024 16:35:34 +0200 Subject: [PATCH 4/6] l0_flush: remove support for mode `page-cached` (#8739) It's been rolled out everywhere, no configs are referencing it. All code that's made dead by the removal of the config option is removed as part of this PR. The `page_caching::PreWarmingWriter` in `::No` mode is equivalent to a `size_tracking_writer`, so, use that. part of https://github.com/neondatabase/neon/issues/7418 --- pageserver/src/l0_flush.rs | 19 +- pageserver/src/tenant/ephemeral_file.rs | 5 +- .../src/tenant/ephemeral_file/page_caching.rs | 169 ++---------------- .../tenant/storage_layer/inmemory_layer.rs | 68 +------ 4 files changed, 20 insertions(+), 241 deletions(-) diff --git a/pageserver/src/l0_flush.rs b/pageserver/src/l0_flush.rs index 10187f2ba3..313a7961a6 100644 --- a/pageserver/src/l0_flush.rs +++ b/pageserver/src/l0_flush.rs @@ -1,15 +1,10 @@ use std::{num::NonZeroUsize, sync::Arc}; -use crate::tenant::ephemeral_file; - #[derive(Debug, PartialEq, Eq, Clone, serde::Deserialize)] #[serde(tag = "mode", rename_all = "kebab-case", deny_unknown_fields)] pub enum L0FlushConfig { - PageCached, #[serde(rename_all = "snake_case")] - Direct { - max_concurrency: NonZeroUsize, - }, + Direct { max_concurrency: NonZeroUsize }, } impl Default for L0FlushConfig { @@ -25,14 +20,12 @@ impl Default for L0FlushConfig { pub struct L0FlushGlobalState(Arc); pub enum Inner { - PageCached, Direct { semaphore: tokio::sync::Semaphore }, } impl L0FlushGlobalState { pub fn new(config: L0FlushConfig) -> Self { match config { - L0FlushConfig::PageCached => Self(Arc::new(Inner::PageCached)), L0FlushConfig::Direct { max_concurrency } => { let semaphore = tokio::sync::Semaphore::new(max_concurrency.get()); Self(Arc::new(Inner::Direct { semaphore })) @@ -44,13 +37,3 @@ impl L0FlushGlobalState { &self.0 } } - -impl L0FlushConfig { - pub(crate) fn prewarm_on_write(&self) -> ephemeral_file::PrewarmPageCacheOnWrite { - use L0FlushConfig::*; - match self { - PageCached => ephemeral_file::PrewarmPageCacheOnWrite::Yes, - Direct { .. } => ephemeral_file::PrewarmPageCacheOnWrite::No, - } - } -} diff --git a/pageserver/src/tenant/ephemeral_file.rs b/pageserver/src/tenant/ephemeral_file.rs index 770f3ca5f0..3eb8384d05 100644 --- a/pageserver/src/tenant/ephemeral_file.rs +++ b/pageserver/src/tenant/ephemeral_file.rs @@ -21,7 +21,6 @@ pub struct EphemeralFile { } mod page_caching; -pub(crate) use page_caching::PrewarmOnWrite as PrewarmPageCacheOnWrite; mod zero_padded_read_write; impl EphemeralFile { @@ -52,12 +51,10 @@ impl EphemeralFile { ) .await?; - let prewarm = conf.l0_flush.prewarm_on_write(); - Ok(EphemeralFile { _tenant_shard_id: tenant_shard_id, _timeline_id: timeline_id, - rw: page_caching::RW::new(file, prewarm, gate_guard), + rw: page_caching::RW::new(file, gate_guard), }) } diff --git a/pageserver/src/tenant/ephemeral_file/page_caching.rs b/pageserver/src/tenant/ephemeral_file/page_caching.rs index 7355b3b5a3..48926354f1 100644 --- a/pageserver/src/tenant/ephemeral_file/page_caching.rs +++ b/pageserver/src/tenant/ephemeral_file/page_caching.rs @@ -1,15 +1,15 @@ //! Wrapper around [`super::zero_padded_read_write::RW`] that uses the //! [`crate::page_cache`] to serve reads that need to go to the underlying [`VirtualFile`]. +//! +//! Subject to removal in use crate::context::RequestContext; use crate::page_cache::{self, PAGE_SZ}; use crate::tenant::block_io::BlockLease; -use crate::virtual_file::owned_buffers_io::io_buf_ext::FullSlice; +use crate::virtual_file::owned_buffers_io::util::size_tracking_writer; use crate::virtual_file::VirtualFile; -use once_cell::sync::Lazy; -use std::io::{self, ErrorKind}; -use std::ops::{Deref, Range}; +use std::io::{self}; use tokio_epoll_uring::BoundedBuf; use tracing::*; @@ -18,33 +18,17 @@ use super::zero_padded_read_write; /// See module-level comment. pub struct RW { page_cache_file_id: page_cache::FileId, - rw: super::zero_padded_read_write::RW, + rw: super::zero_padded_read_write::RW>, /// Gate guard is held on as long as we need to do operations in the path (delete on drop). _gate_guard: utils::sync::gate::GateGuard, } -/// When we flush a block to the underlying [`crate::virtual_file::VirtualFile`], -/// should we pre-warm the [`crate::page_cache`] with the contents? -#[derive(Clone, Copy)] -pub enum PrewarmOnWrite { - Yes, - No, -} - impl RW { - pub fn new( - file: VirtualFile, - prewarm_on_write: PrewarmOnWrite, - _gate_guard: utils::sync::gate::GateGuard, - ) -> Self { + pub fn new(file: VirtualFile, _gate_guard: utils::sync::gate::GateGuard) -> Self { let page_cache_file_id = page_cache::next_file_id(); Self { page_cache_file_id, - rw: super::zero_padded_read_write::RW::new(PreWarmingWriter::new( - page_cache_file_id, - file, - prewarm_on_write, - )), + rw: super::zero_padded_read_write::RW::new(size_tracking_writer::Writer::new(file)), _gate_guard, } } @@ -84,10 +68,10 @@ impl RW { let vec = Vec::with_capacity(size); // read from disk what we've already flushed - let writer = self.rw.as_writer(); - let flushed_range = writer.written_range(); - let mut vec = writer - .file + let file_size_tracking_writer = self.rw.as_writer(); + let flushed_range = 0..usize::try_from(file_size_tracking_writer.bytes_written()).unwrap(); + let mut vec = file_size_tracking_writer + .as_inner() .read_exact_at( vec.slice(0..(flushed_range.end - flushed_range.start)), u64::try_from(flushed_range.start).unwrap(), @@ -122,7 +106,7 @@ impl RW { format!( "ephemeral file: read immutable page #{}: {}: {:#}", blknum, - self.rw.as_writer().file.path, + self.rw.as_writer().as_inner().path, e, ), ) @@ -132,7 +116,7 @@ impl RW { } page_cache::ReadBufResult::NotFound(write_guard) => { let write_guard = writer - .file + .as_inner() .read_exact_at_page(write_guard, blknum as u64 * PAGE_SZ as u64, ctx) .await?; let read_guard = write_guard.mark_valid(); @@ -154,137 +138,16 @@ impl Drop for RW { // unlink the file // we are clear to do this, because we have entered a gate - let res = std::fs::remove_file(&self.rw.as_writer().file.path); + let path = &self.rw.as_writer().as_inner().path; + let res = std::fs::remove_file(path); if let Err(e) = res { if e.kind() != std::io::ErrorKind::NotFound { // just never log the not found errors, we cannot do anything for them; on detach // the tenant directory is already gone. // // not found files might also be related to https://github.com/neondatabase/neon/issues/2442 - error!( - "could not remove ephemeral file '{}': {}", - self.rw.as_writer().file.path, - e - ); + error!("could not remove ephemeral file '{path}': {e}"); } } } } - -struct PreWarmingWriter { - prewarm_on_write: PrewarmOnWrite, - nwritten_blocks: u32, - page_cache_file_id: page_cache::FileId, - file: VirtualFile, -} - -impl PreWarmingWriter { - fn new( - page_cache_file_id: page_cache::FileId, - file: VirtualFile, - prewarm_on_write: PrewarmOnWrite, - ) -> Self { - Self { - prewarm_on_write, - nwritten_blocks: 0, - page_cache_file_id, - file, - } - } - - /// Return the byte range within `file` that has been written though `write_all`. - /// - /// The returned range would be invalidated by another `write_all`. To prevent that, we capture `&_`. - fn written_range(&self) -> (impl Deref> + '_) { - let nwritten_blocks = usize::try_from(self.nwritten_blocks).unwrap(); - struct Wrapper(Range); - impl Deref for Wrapper { - type Target = Range; - fn deref(&self) -> &Range { - &self.0 - } - } - Wrapper(0..nwritten_blocks * PAGE_SZ) - } -} - -impl crate::virtual_file::owned_buffers_io::write::OwnedAsyncWriter for PreWarmingWriter { - async fn write_all( - &mut self, - buf: FullSlice, - ctx: &RequestContext, - ) -> std::io::Result<(usize, FullSlice)> { - let buflen = buf.len(); - assert_eq!( - buflen % PAGE_SZ, - 0, - "{buflen} ; we know TAIL_SZ is a PAGE_SZ multiple, and write_buffered_borrowed is used" - ); - - // Do the IO. - let buf = match self.file.write_all(buf, ctx).await { - (buf, Ok(nwritten)) => { - assert_eq!(nwritten, buflen); - buf - } - (_, Err(e)) => { - return Err(std::io::Error::new( - ErrorKind::Other, - // order error before path because path is long and error is short - format!( - "ephemeral_file: write_blob: write-back tail self.nwritten_blocks={}, buflen={}, {:#}: {}", - self.nwritten_blocks, buflen, e, self.file.path, - ), - )); - } - }; - - let nblocks = buflen / PAGE_SZ; - let nblocks32 = u32::try_from(nblocks).unwrap(); - - if matches!(self.prewarm_on_write, PrewarmOnWrite::Yes) { - // Pre-warm page cache with the contents. - // At least in isolated bulk ingest benchmarks (test_bulk_insert.py), the pre-warming - // benefits the code that writes InMemoryLayer=>L0 layers. - - let cache = page_cache::get(); - static CTX: Lazy = Lazy::new(|| { - RequestContext::new( - crate::task_mgr::TaskKind::EphemeralFilePreWarmPageCache, - crate::context::DownloadBehavior::Error, - ) - }); - for blknum_in_buffer in 0..nblocks { - let blk_in_buffer = - &buf[blknum_in_buffer * PAGE_SZ..(blknum_in_buffer + 1) * PAGE_SZ]; - let blknum = self - .nwritten_blocks - .checked_add(blknum_in_buffer as u32) - .unwrap(); - match cache - .read_immutable_buf(self.page_cache_file_id, blknum, &CTX) - .await - { - Err(e) => { - error!("ephemeral_file write_blob failed to get immutable buf to pre-warm page cache: {e:?}"); - // fail gracefully, it's not the end of the world if we can't pre-warm the cache here - } - Ok(v) => match v { - page_cache::ReadBufResult::Found(_guard) => { - // This function takes &mut self, so, it shouldn't be possible to reach this point. - unreachable!("we just wrote block {blknum} to the VirtualFile, which is owned by Self, \ - and this function takes &mut self, so, no concurrent read_blk is possible"); - } - page_cache::ReadBufResult::NotFound(mut write_guard) => { - write_guard.copy_from_slice(blk_in_buffer); - let _ = write_guard.mark_valid(); - } - }, - } - } - } - - self.nwritten_blocks = self.nwritten_blocks.checked_add(nblocks32).unwrap(); - Ok((buflen, buf)) - } -} diff --git a/pageserver/src/tenant/storage_layer/inmemory_layer.rs b/pageserver/src/tenant/storage_layer/inmemory_layer.rs index 748d79c149..130d1002a0 100644 --- a/pageserver/src/tenant/storage_layer/inmemory_layer.rs +++ b/pageserver/src/tenant/storage_layer/inmemory_layer.rs @@ -13,7 +13,7 @@ use crate::tenant::ephemeral_file::EphemeralFile; use crate::tenant::timeline::GetVectoredError; use crate::tenant::PageReconstructError; use crate::virtual_file::owned_buffers_io::io_buf_ext::IoBufExt; -use crate::{l0_flush, page_cache, walrecord}; +use crate::{l0_flush, page_cache}; use anyhow::{anyhow, Result}; use camino::Utf8PathBuf; use pageserver_api::key::CompactKey; @@ -249,9 +249,7 @@ impl InMemoryLayer { /// debugging function to print out the contents of the layer /// /// this is likely completly unused - pub async fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()> { - let inner = self.inner.read().await; - + pub async fn dump(&self, _verbose: bool, _ctx: &RequestContext) -> Result<()> { let end_str = self.end_lsn_or_max(); println!( @@ -259,39 +257,6 @@ impl InMemoryLayer { self.timeline_id, self.start_lsn, end_str, ); - if !verbose { - return Ok(()); - } - - let cursor = inner.file.block_cursor(); - let mut buf = Vec::new(); - for (key, vec_map) in inner.index.iter() { - for (lsn, pos) in vec_map.as_slice() { - let mut desc = String::new(); - cursor.read_blob_into_buf(*pos, &mut buf, ctx).await?; - let val = Value::des(&buf); - match val { - Ok(Value::Image(img)) => { - write!(&mut desc, " img {} bytes", img.len())?; - } - Ok(Value::WalRecord(rec)) => { - let wal_desc = walrecord::describe_wal_record(&rec).unwrap(); - write!( - &mut desc, - " rec {} bytes will_init: {} {}", - buf.len(), - rec.will_init(), - wal_desc - )?; - } - Err(err) => { - write!(&mut desc, " DESERIALIZATION ERROR: {}", err)?; - } - } - println!(" key {} at {}: {}", key, lsn, desc); - } - } - Ok(()) } @@ -536,7 +501,6 @@ impl InMemoryLayer { use l0_flush::Inner; let _concurrency_permit = match l0_flush_global_state { - Inner::PageCached => None, Inner::Direct { semaphore, .. } => Some(semaphore.acquire().await), }; @@ -568,34 +532,6 @@ impl InMemoryLayer { .await?; match l0_flush_global_state { - l0_flush::Inner::PageCached => { - let ctx = RequestContextBuilder::extend(ctx) - .page_content_kind(PageContentKind::InMemoryLayer) - .build(); - - let mut buf = Vec::new(); - - let cursor = inner.file.block_cursor(); - - for (key, vec_map) in inner.index.iter() { - // Write all page versions - for (lsn, pos) in vec_map.as_slice() { - cursor.read_blob_into_buf(*pos, &mut buf, &ctx).await?; - let will_init = Value::des(&buf)?.will_init(); - let (tmp, res) = delta_layer_writer - .put_value_bytes( - Key::from_compact(*key), - *lsn, - buf.slice_len(), - will_init, - &ctx, - ) - .await; - res?; - buf = tmp.into_raw_slice().into_inner(); - } - } - } l0_flush::Inner::Direct { .. } => { let file_contents: Vec = inner.file.load_to_vec(ctx).await?; assert_eq!( From 3b8ca477ab6852143f8acb5b8217e5f24e9e8605 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Mon, 19 Aug 2024 16:39:44 +0200 Subject: [PATCH 5/6] Migrate physical GC and scan_metadata to remote_storage (#8673) Migrates most of the remaining parts of the scrubber to remote_storage: * `pageserver_physical_gc` * `scan_metadata` for pageservers (safekeepers were done in #8595) * `download()` in `tenant_snapshot`. The main `tenant_snapshot` is not migrated as it uses version history to be able to work in the face of ongoing changes. Part of #7547 --- libs/remote_storage/src/azure_blob.rs | 42 +++++ libs/remote_storage/src/lib.rs | 24 ++- libs/remote_storage/src/local_fs.rs | 14 ++ libs/remote_storage/src/metrics.rs | 9 +- libs/remote_storage/src/s3_bucket.rs | 74 +++++++- libs/remote_storage/src/simulate_failures.rs | 11 ++ storage_scrubber/src/checks.rs | 85 ++++----- storage_scrubber/src/lib.rs | 69 ++++--- storage_scrubber/src/metadata_stream.rs | 170 ++++-------------- .../src/pageserver_physical_gc.rs | 134 +++++--------- .../src/scan_pageserver_metadata.rs | 40 ++--- .../src/scan_safekeeper_metadata.rs | 7 +- storage_scrubber/src/tenant_snapshot.rs | 27 +-- test_runner/fixtures/neon_fixtures.py | 1 + 14 files changed, 366 insertions(+), 341 deletions(-) diff --git a/libs/remote_storage/src/azure_blob.rs b/libs/remote_storage/src/azure_blob.rs index 3c77d5a227..cb7479f6cd 100644 --- a/libs/remote_storage/src/azure_blob.rs +++ b/libs/remote_storage/src/azure_blob.rs @@ -383,6 +383,48 @@ impl RemoteStorage for AzureBlobStorage { } } + async fn head_object( + &self, + key: &RemotePath, + cancel: &CancellationToken, + ) -> Result { + let kind = RequestKind::Head; + let _permit = self.permit(kind, cancel).await?; + + let started_at = start_measuring_requests(kind); + + let blob_client = self.client.blob_client(self.relative_path_to_name(key)); + let properties_future = blob_client.get_properties().into_future(); + + let properties_future = tokio::time::timeout(self.timeout, properties_future); + + let res = tokio::select! { + res = properties_future => res, + _ = cancel.cancelled() => return Err(TimeoutOrCancel::Cancel.into()), + }; + + if let Ok(inner) = &res { + // do not incl. timeouts as errors in metrics but cancellations + let started_at = ScopeGuard::into_inner(started_at); + crate::metrics::BUCKET_METRICS + .req_seconds + .observe_elapsed(kind, inner, started_at); + } + + let data = match res { + Ok(Ok(data)) => Ok(data), + Ok(Err(sdk)) => Err(to_download_error(sdk)), + Err(_timeout) => Err(DownloadError::Timeout), + }?; + + let properties = data.blob.properties; + Ok(ListingObject { + key: key.to_owned(), + last_modified: SystemTime::from(properties.last_modified), + size: properties.content_length, + }) + } + async fn upload( &self, from: impl Stream> + Send + Sync + 'static, diff --git a/libs/remote_storage/src/lib.rs b/libs/remote_storage/src/lib.rs index 2c9e298f79..cc1d3e0ae4 100644 --- a/libs/remote_storage/src/lib.rs +++ b/libs/remote_storage/src/lib.rs @@ -150,7 +150,7 @@ pub enum ListingMode { NoDelimiter, } -#[derive(PartialEq, Eq, Debug)] +#[derive(PartialEq, Eq, Debug, Clone)] pub struct ListingObject { pub key: RemotePath, pub last_modified: SystemTime, @@ -215,6 +215,13 @@ pub trait RemoteStorage: Send + Sync + 'static { Ok(combined) } + /// Obtain metadata information about an object. + async fn head_object( + &self, + key: &RemotePath, + cancel: &CancellationToken, + ) -> Result; + /// Streams the local file contents into remote into the remote storage entry. /// /// If the operation fails because of timeout or cancellation, the root cause of the error will be @@ -363,6 +370,20 @@ impl GenericRemoteStorage> { } } + // See [`RemoteStorage::head_object`]. + pub async fn head_object( + &self, + key: &RemotePath, + cancel: &CancellationToken, + ) -> Result { + match self { + Self::LocalFs(s) => s.head_object(key, cancel).await, + Self::AwsS3(s) => s.head_object(key, cancel).await, + Self::AzureBlob(s) => s.head_object(key, cancel).await, + Self::Unreliable(s) => s.head_object(key, cancel).await, + } + } + /// See [`RemoteStorage::upload`] pub async fn upload( &self, @@ -598,6 +619,7 @@ impl ConcurrencyLimiter { RequestKind::Delete => &self.write, RequestKind::Copy => &self.write, RequestKind::TimeTravel => &self.write, + RequestKind::Head => &self.read, } } diff --git a/libs/remote_storage/src/local_fs.rs b/libs/remote_storage/src/local_fs.rs index 99b4aa4061..c3ef18cab1 100644 --- a/libs/remote_storage/src/local_fs.rs +++ b/libs/remote_storage/src/local_fs.rs @@ -445,6 +445,20 @@ impl RemoteStorage for LocalFs { } } + async fn head_object( + &self, + key: &RemotePath, + _cancel: &CancellationToken, + ) -> Result { + let target_file_path = key.with_base(&self.storage_root); + let metadata = file_metadata(&target_file_path).await?; + Ok(ListingObject { + key: key.clone(), + last_modified: metadata.modified()?, + size: metadata.len(), + }) + } + async fn upload( &self, data: impl Stream> + Send + Sync, diff --git a/libs/remote_storage/src/metrics.rs b/libs/remote_storage/src/metrics.rs index bbb51590f3..f1aa4c433b 100644 --- a/libs/remote_storage/src/metrics.rs +++ b/libs/remote_storage/src/metrics.rs @@ -13,6 +13,7 @@ pub(crate) enum RequestKind { List = 3, Copy = 4, TimeTravel = 5, + Head = 6, } use scopeguard::ScopeGuard; @@ -27,6 +28,7 @@ impl RequestKind { List => "list_objects", Copy => "copy_object", TimeTravel => "time_travel_recover", + Head => "head_object", } } const fn as_index(&self) -> usize { @@ -34,7 +36,8 @@ impl RequestKind { } } -pub(crate) struct RequestTyped([C; 6]); +const REQUEST_KIND_COUNT: usize = 7; +pub(crate) struct RequestTyped([C; REQUEST_KIND_COUNT]); impl RequestTyped { pub(crate) fn get(&self, kind: RequestKind) -> &C { @@ -43,8 +46,8 @@ impl RequestTyped { fn build_with(mut f: impl FnMut(RequestKind) -> C) -> Self { use RequestKind::*; - let mut it = [Get, Put, Delete, List, Copy, TimeTravel].into_iter(); - let arr = std::array::from_fn::(|index| { + let mut it = [Get, Put, Delete, List, Copy, TimeTravel, Head].into_iter(); + let arr = std::array::from_fn::(|index| { let next = it.next().unwrap(); assert_eq!(index, next.as_index()); f(next) diff --git a/libs/remote_storage/src/s3_bucket.rs b/libs/remote_storage/src/s3_bucket.rs index 1f25da813d..11f6598cbf 100644 --- a/libs/remote_storage/src/s3_bucket.rs +++ b/libs/remote_storage/src/s3_bucket.rs @@ -23,7 +23,7 @@ use aws_config::{ use aws_sdk_s3::{ config::{AsyncSleep, IdentityCache, Region, SharedAsyncSleep}, error::SdkError, - operation::get_object::GetObjectError, + operation::{get_object::GetObjectError, head_object::HeadObjectError}, types::{Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion, StorageClass}, Client, }; @@ -604,6 +604,78 @@ impl RemoteStorage for S3Bucket { } } + async fn head_object( + &self, + key: &RemotePath, + cancel: &CancellationToken, + ) -> Result { + let kind = RequestKind::Head; + let _permit = self.permit(kind, cancel).await?; + + let started_at = start_measuring_requests(kind); + + let head_future = self + .client + .head_object() + .bucket(self.bucket_name()) + .key(self.relative_path_to_s3_object(key)) + .send(); + + let head_future = tokio::time::timeout(self.timeout, head_future); + + let res = tokio::select! { + res = head_future => res, + _ = cancel.cancelled() => return Err(TimeoutOrCancel::Cancel.into()), + }; + + let res = res.map_err(|_e| DownloadError::Timeout)?; + + // do not incl. timeouts as errors in metrics but cancellations + let started_at = ScopeGuard::into_inner(started_at); + crate::metrics::BUCKET_METRICS + .req_seconds + .observe_elapsed(kind, &res, started_at); + + let data = match res { + Ok(object_output) => object_output, + Err(SdkError::ServiceError(e)) if matches!(e.err(), HeadObjectError::NotFound(_)) => { + // Count this in the AttemptOutcome::Ok bucket, because 404 is not + // an error: we expect to sometimes fetch an object and find it missing, + // e.g. when probing for timeline indices. + crate::metrics::BUCKET_METRICS.req_seconds.observe_elapsed( + kind, + AttemptOutcome::Ok, + started_at, + ); + return Err(DownloadError::NotFound); + } + Err(e) => { + crate::metrics::BUCKET_METRICS.req_seconds.observe_elapsed( + kind, + AttemptOutcome::Err, + started_at, + ); + + return Err(DownloadError::Other( + anyhow::Error::new(e).context("s3 head object"), + )); + } + }; + + let (Some(last_modified), Some(size)) = (data.last_modified, data.content_length) else { + return Err(DownloadError::Other(anyhow!( + "head_object doesn't contain last_modified or content_length" + )))?; + }; + Ok(ListingObject { + key: key.to_owned(), + last_modified: SystemTime::try_from(last_modified).map_err(|e| { + DownloadError::Other(anyhow!("can't convert time '{last_modified}': {e}")) + })?, + size: size as u64, + }) + } + async fn upload( &self, from: impl Stream> + Send + Sync + 'static, diff --git a/libs/remote_storage/src/simulate_failures.rs b/libs/remote_storage/src/simulate_failures.rs index 13f873dcdb..c7eb634af3 100644 --- a/libs/remote_storage/src/simulate_failures.rs +++ b/libs/remote_storage/src/simulate_failures.rs @@ -30,6 +30,7 @@ pub struct UnreliableWrapper { #[derive(Debug, Hash, Eq, PartialEq)] enum RemoteOp { ListPrefixes(Option), + HeadObject(RemotePath), Upload(RemotePath), Download(RemotePath), Delete(RemotePath), @@ -137,6 +138,16 @@ impl RemoteStorage for UnreliableWrapper { self.inner.list(prefix, mode, max_keys, cancel).await } + async fn head_object( + &self, + key: &RemotePath, + cancel: &CancellationToken, + ) -> Result { + self.attempt(RemoteOp::HeadObject(key.clone())) + .map_err(DownloadError::Other)?; + self.inner.head_object(key, cancel).await + } + async fn upload( &self, data: impl Stream> + Send + Sync + 'static, diff --git a/storage_scrubber/src/checks.rs b/storage_scrubber/src/checks.rs index 35ec69fd50..9063b3c197 100644 --- a/storage_scrubber/src/checks.rs +++ b/storage_scrubber/src/checks.rs @@ -1,22 +1,22 @@ use std::collections::{HashMap, HashSet}; use anyhow::Context; -use aws_sdk_s3::Client; use pageserver::tenant::layer_map::LayerMap; use pageserver::tenant::remote_timeline_client::index::LayerFileMetadata; use pageserver_api::shard::ShardIndex; +use tokio_util::sync::CancellationToken; use tracing::{error, info, warn}; use utils::generation::Generation; use utils::id::TimelineId; use crate::cloud_admin_api::BranchData; -use crate::metadata_stream::stream_listing; -use crate::{download_object_with_retries, RootTarget, TenantShardTimelineId}; +use crate::metadata_stream::stream_listing_generic; +use crate::{download_object_with_retries_generic, RootTarget, TenantShardTimelineId}; use futures_util::StreamExt; use pageserver::tenant::remote_timeline_client::{parse_remote_index_path, remote_layer_path}; use pageserver::tenant::storage_layer::LayerName; use pageserver::tenant::IndexPart; -use remote_storage::RemotePath; +use remote_storage::{GenericRemoteStorage, ListingObject, RemotePath}; pub(crate) struct TimelineAnalysis { /// Anomalies detected @@ -48,13 +48,12 @@ impl TimelineAnalysis { } pub(crate) async fn branch_cleanup_and_check_errors( - s3_client: &Client, - target: &RootTarget, + remote_client: &GenericRemoteStorage, id: &TenantShardTimelineId, tenant_objects: &mut TenantObjectListing, s3_active_branch: Option<&BranchData>, console_branch: Option, - s3_data: Option, + s3_data: Option, ) -> TimelineAnalysis { let mut result = TimelineAnalysis::new(); @@ -78,7 +77,9 @@ pub(crate) async fn branch_cleanup_and_check_errors( match s3_data { Some(s3_data) => { - result.garbage_keys.extend(s3_data.unknown_keys); + result + .garbage_keys + .extend(s3_data.unknown_keys.into_iter().map(|k| k.key.to_string())); match s3_data.blob_data { BlobDataParseResult::Parsed { @@ -143,11 +144,8 @@ pub(crate) async fn branch_cleanup_and_check_errors( // HEAD request used here to address a race condition when an index was uploaded concurrently // with our scan. We check if the object is uploaded to S3 after taking the listing snapshot. - let response = s3_client - .head_object() - .bucket(target.bucket_name()) - .key(path.get_path().as_str()) - .send() + let response = remote_client + .head_object(&path, &CancellationToken::new()) .await; if response.is_err() { @@ -284,14 +282,14 @@ impl TenantObjectListing { } #[derive(Debug)] -pub(crate) struct S3TimelineBlobData { +pub(crate) struct RemoteTimelineBlobData { pub(crate) blob_data: BlobDataParseResult, // Index objects that were not used when loading `blob_data`, e.g. those from old generations - pub(crate) unused_index_keys: Vec, + pub(crate) unused_index_keys: Vec, // Objects whose keys were not recognized at all, i.e. not layer files, not indices - pub(crate) unknown_keys: Vec, + pub(crate) unknown_keys: Vec, } #[derive(Debug)] @@ -322,11 +320,11 @@ pub(crate) fn parse_layer_object_name(name: &str) -> Result<(LayerName, Generati } } -pub(crate) async fn list_timeline_blobs( - s3_client: &Client, +pub(crate) async fn list_timeline_blobs_generic( + remote_client: &GenericRemoteStorage, id: TenantShardTimelineId, s3_root: &RootTarget, -) -> anyhow::Result { +) -> anyhow::Result { let mut s3_layers = HashSet::new(); let mut errors = Vec::new(); @@ -335,19 +333,25 @@ pub(crate) async fn list_timeline_blobs( let mut timeline_dir_target = s3_root.timeline_root(&id); timeline_dir_target.delimiter = String::new(); - let mut index_part_keys: Vec = Vec::new(); + let mut index_part_keys: Vec = Vec::new(); let mut initdb_archive: bool = false; - let mut stream = std::pin::pin!(stream_listing(s3_client, &timeline_dir_target)); - while let Some(obj) = stream.next().await { - let obj = obj?; - let key = obj.key(); + let prefix_str = &timeline_dir_target + .prefix_in_bucket + .strip_prefix("/") + .unwrap_or(&timeline_dir_target.prefix_in_bucket); - let blob_name = key.strip_prefix(&timeline_dir_target.prefix_in_bucket); + let mut stream = std::pin::pin!(stream_listing_generic(remote_client, &timeline_dir_target)); + while let Some(obj) = stream.next().await { + let (key, Some(obj)) = obj? else { + panic!("ListingObject not specified"); + }; + + let blob_name = key.get_path().as_str().strip_prefix(prefix_str); match blob_name { Some(name) if name.starts_with("index_part.json") => { tracing::debug!("Index key {key}"); - index_part_keys.push(key.to_owned()) + index_part_keys.push(obj) } Some("initdb.tar.zst") => { tracing::debug!("initdb archive {key}"); @@ -358,7 +362,7 @@ pub(crate) async fn list_timeline_blobs( } Some(maybe_layer_name) => match parse_layer_object_name(maybe_layer_name) { Ok((new_layer, gen)) => { - tracing::debug!("Parsed layer key: {} {:?}", new_layer, gen); + tracing::debug!("Parsed layer key: {new_layer} {gen:?}"); s3_layers.insert((new_layer, gen)); } Err(e) => { @@ -366,13 +370,13 @@ pub(crate) async fn list_timeline_blobs( errors.push( format!("S3 list response got an object with key {key} that is not a layer name: {e}"), ); - unknown_keys.push(key.to_string()); + unknown_keys.push(obj); } }, None => { - tracing::warn!("Unknown key {}", key); + tracing::warn!("Unknown key {key}"); errors.push(format!("S3 list response got an object with odd key {key}")); - unknown_keys.push(key.to_string()); + unknown_keys.push(obj); } } } @@ -381,7 +385,7 @@ pub(crate) async fn list_timeline_blobs( tracing::debug!( "Timeline is empty apart from initdb archive: expected post-deletion state." ); - return Ok(S3TimelineBlobData { + return Ok(RemoteTimelineBlobData { blob_data: BlobDataParseResult::Relic, unused_index_keys: index_part_keys, unknown_keys: Vec::new(), @@ -395,13 +399,13 @@ pub(crate) async fn list_timeline_blobs( // Stripping the index key to the last part, because RemotePath doesn't // like absolute paths, and depending on prefix_in_bucket it's possible // for the keys we read back to start with a slash. - let basename = key.rsplit_once('/').unwrap().1; + let basename = key.key.get_path().as_str().rsplit_once('/').unwrap().1; parse_remote_index_path(RemotePath::from_string(basename).unwrap()).map(|g| (key, g)) }) .max_by_key(|i| i.1) .map(|(k, g)| (k.clone(), g)) { - Some((key, gen)) => (Some(key), gen), + Some((key, gen)) => (Some::(key.to_owned()), gen), None => { // Legacy/missing case: one or zero index parts, which did not have a generation (index_part_keys.pop(), Generation::none()) @@ -416,17 +420,14 @@ pub(crate) async fn list_timeline_blobs( } if let Some(index_part_object_key) = index_part_object.as_ref() { - let index_part_bytes = download_object_with_retries( - s3_client, - &timeline_dir_target.bucket_name, - index_part_object_key, - ) - .await - .context("index_part.json download")?; + let index_part_bytes = + download_object_with_retries_generic(remote_client, &index_part_object_key.key) + .await + .context("index_part.json download")?; match serde_json::from_slice(&index_part_bytes) { Ok(index_part) => { - return Ok(S3TimelineBlobData { + return Ok(RemoteTimelineBlobData { blob_data: BlobDataParseResult::Parsed { index_part: Box::new(index_part), index_part_generation, @@ -448,7 +449,7 @@ pub(crate) async fn list_timeline_blobs( ); } - Ok(S3TimelineBlobData { + Ok(RemoteTimelineBlobData { blob_data: BlobDataParseResult::Incorrect { errors, s3_layers }, unused_index_keys: index_part_keys, unknown_keys, diff --git a/storage_scrubber/src/lib.rs b/storage_scrubber/src/lib.rs index 1fc94cc174..3183bc3c64 100644 --- a/storage_scrubber/src/lib.rs +++ b/storage_scrubber/src/lib.rs @@ -452,23 +452,26 @@ fn stream_objects_with_retries<'a>( let mut list_stream = storage_client.list_streaming(Some(&prefix), listing_mode, None, &cancel); while let Some(res) = list_stream.next().await { - if let Err(err) = res { - let yield_err = if err.is_permanent() { - true - } else { - let backoff_time = 1 << trial.max(5); - tokio::time::sleep(Duration::from_secs(backoff_time)).await; - trial += 1; - trial == MAX_RETRIES - 1 - }; - if yield_err { - yield Err(err) - .with_context(|| format!("Failed to list objects {MAX_RETRIES} times")); - break; + match res { + Err(err) => { + let yield_err = if err.is_permanent() { + true + } else { + let backoff_time = 1 << trial.max(5); + tokio::time::sleep(Duration::from_secs(backoff_time)).await; + trial += 1; + trial == MAX_RETRIES - 1 + }; + if yield_err { + yield Err(err) + .with_context(|| format!("Failed to list objects {MAX_RETRIES} times")); + break; + } + } + Ok(res) => { + trial = 0; + yield Ok(res); } - } else { - trial = 0; - yield res.map_err(anyhow::Error::from); } } } @@ -513,41 +516,35 @@ async fn list_objects_with_retries_generic( panic!("MAX_RETRIES is not allowed to be 0"); } -async fn download_object_with_retries( - s3_client: &Client, - bucket_name: &str, - key: &str, +async fn download_object_with_retries_generic( + remote_client: &GenericRemoteStorage, + key: &RemotePath, ) -> anyhow::Result> { - for _ in 0..MAX_RETRIES { - let mut body_buf = Vec::new(); - let response_stream = match s3_client - .get_object() - .bucket(bucket_name) - .key(key) - .send() - .await - { + let cancel = CancellationToken::new(); + for trial in 0..MAX_RETRIES { + let mut buf = Vec::new(); + let download = match remote_client.download(key, &cancel).await { Ok(response) => response, Err(e) => { error!("Failed to download object for key {key}: {e}"); - tokio::time::sleep(Duration::from_secs(1)).await; + let backoff_time = 1 << trial.max(5); + tokio::time::sleep(Duration::from_secs(backoff_time)).await; continue; } }; - match response_stream - .body - .into_async_read() - .read_to_end(&mut body_buf) + match tokio_util::io::StreamReader::new(download.download_stream) + .read_to_end(&mut buf) .await { Ok(bytes_read) => { tracing::debug!("Downloaded {bytes_read} bytes for object {key}"); - return Ok(body_buf); + return Ok(buf); } Err(e) => { error!("Failed to stream object body for key {key}: {e}"); - tokio::time::sleep(Duration::from_secs(1)).await; + let backoff_time = 1 << trial.max(5); + tokio::time::sleep(Duration::from_secs(backoff_time)).await; } } } diff --git a/storage_scrubber/src/metadata_stream.rs b/storage_scrubber/src/metadata_stream.rs index 54812ffc94..eca774413a 100644 --- a/storage_scrubber/src/metadata_stream.rs +++ b/storage_scrubber/src/metadata_stream.rs @@ -2,14 +2,14 @@ use std::str::FromStr; use anyhow::{anyhow, Context}; use async_stream::{stream, try_stream}; -use aws_sdk_s3::{types::ObjectIdentifier, Client}; +use aws_sdk_s3::Client; use futures::StreamExt; use remote_storage::{GenericRemoteStorage, ListingMode, ListingObject, RemotePath}; use tokio_stream::Stream; use crate::{ - list_objects_with_retries, stream_objects_with_retries, RootTarget, S3Target, - TenantShardTimelineId, + list_objects_with_retries, list_objects_with_retries_generic, stream_objects_with_retries, + RootTarget, S3Target, TenantShardTimelineId, }; use pageserver_api::shard::TenantShardId; use utils::id::{TenantId, TimelineId}; @@ -75,53 +75,38 @@ pub fn stream_tenants<'a>( } pub async fn stream_tenant_shards<'a>( - s3_client: &'a Client, + remote_client: &'a GenericRemoteStorage, target: &'a RootTarget, tenant_id: TenantId, ) -> anyhow::Result> + 'a> { - let mut tenant_shard_ids: Vec> = Vec::new(); - let mut continuation_token = None; let shards_target = target.tenant_shards_prefix(&tenant_id); - loop { - tracing::info!("Listing in {}", shards_target.prefix_in_bucket); - let fetch_response = - list_objects_with_retries(s3_client, &shards_target, continuation_token.clone()).await; - let fetch_response = match fetch_response { - Err(e) => { - tenant_shard_ids.push(Err(e)); - break; - } - Ok(r) => r, - }; + let strip_prefix = target.tenants_root().prefix_in_bucket; + let prefix_str = &strip_prefix.strip_prefix("/").unwrap_or(&strip_prefix); - let new_entry_ids = fetch_response - .common_prefixes() - .iter() - .filter_map(|prefix| prefix.prefix()) - .filter_map(|prefix| -> Option<&str> { - prefix - .strip_prefix(&target.tenants_root().prefix_in_bucket)? - .strip_suffix('/') - }) - .map(|entry_id_str| { - let first_part = entry_id_str.split('/').next().unwrap(); + tracing::info!("Listing shards in {}", shards_target.prefix_in_bucket); + let listing = list_objects_with_retries_generic( + remote_client, + ListingMode::WithDelimiter, + &shards_target, + ) + .await?; - first_part - .parse::() - .with_context(|| format!("Incorrect entry id str: {first_part}")) - }); + let tenant_shard_ids = listing + .prefixes + .iter() + .map(|prefix| prefix.get_path().as_str()) + .filter_map(|prefix| -> Option<&str> { prefix.strip_prefix(prefix_str) }) + .map(|entry_id_str| { + let first_part = entry_id_str.split('/').next().unwrap(); - for i in new_entry_ids { - tenant_shard_ids.push(i); - } - - match fetch_response.next_continuation_token { - Some(new_token) => continuation_token = Some(new_token), - None => break, - } - } + first_part + .parse::() + .with_context(|| format!("Incorrect entry id str: {first_part}")) + }) + .collect::>(); + tracing::debug!("Yielding {} shards for {tenant_id}", tenant_shard_ids.len()); Ok(stream! { for i in tenant_shard_ids { let id = i?; @@ -130,65 +115,6 @@ pub async fn stream_tenant_shards<'a>( }) } -/// Given a TenantShardId, output a stream of the timelines within that tenant, discovered -/// using ListObjectsv2. The listing is done before the stream is built, so that this -/// function can be used to generate concurrency on a stream using buffer_unordered. -pub async fn stream_tenant_timelines<'a>( - s3_client: &'a Client, - target: &'a RootTarget, - tenant: TenantShardId, -) -> anyhow::Result> + 'a> { - let mut timeline_ids: Vec> = Vec::new(); - let mut continuation_token = None; - let timelines_target = target.timelines_root(&tenant); - - loop { - tracing::debug!("Listing in {}", tenant); - let fetch_response = - list_objects_with_retries(s3_client, &timelines_target, continuation_token.clone()) - .await; - let fetch_response = match fetch_response { - Err(e) => { - timeline_ids.push(Err(e)); - break; - } - Ok(r) => r, - }; - - let new_entry_ids = fetch_response - .common_prefixes() - .iter() - .filter_map(|prefix| prefix.prefix()) - .filter_map(|prefix| -> Option<&str> { - prefix - .strip_prefix(&timelines_target.prefix_in_bucket)? - .strip_suffix('/') - }) - .map(|entry_id_str| { - entry_id_str - .parse::() - .with_context(|| format!("Incorrect entry id str: {entry_id_str}")) - }); - - for i in new_entry_ids { - timeline_ids.push(i); - } - - match fetch_response.next_continuation_token { - Some(new_token) => continuation_token = Some(new_token), - None => break, - } - } - - tracing::debug!("Yielding for {}", tenant); - Ok(stream! { - for i in timeline_ids { - let id = i?; - yield Ok(TenantShardTimelineId::new(tenant, id)); - } - }) -} - /// Given a `TenantShardId`, output a stream of the timelines within that tenant, discovered /// using a listing. The listing is done before the stream is built, so that this /// function can be used to generate concurrency on a stream using buffer_unordered. @@ -200,6 +126,11 @@ pub async fn stream_tenant_timelines_generic<'a>( let mut timeline_ids: Vec> = Vec::new(); let timelines_target = target.timelines_root(&tenant); + let prefix_str = &timelines_target + .prefix_in_bucket + .strip_prefix("/") + .unwrap_or(&timelines_target.prefix_in_bucket); + let mut objects_stream = std::pin::pin!(stream_objects_with_retries( remote_client, ListingMode::WithDelimiter, @@ -220,11 +151,7 @@ pub async fn stream_tenant_timelines_generic<'a>( .prefixes .iter() .filter_map(|prefix| -> Option<&str> { - prefix - .get_path() - .as_str() - .strip_prefix(&timelines_target.prefix_in_bucket)? - .strip_suffix('/') + prefix.get_path().as_str().strip_prefix(prefix_str) }) .map(|entry_id_str| { entry_id_str @@ -237,7 +164,7 @@ pub async fn stream_tenant_timelines_generic<'a>( } } - tracing::debug!("Yielding for {}", tenant); + tracing::debug!("Yielding {} timelines for {}", timeline_ids.len(), tenant); Ok(stream! { for i in timeline_ids { let id = i?; @@ -246,37 +173,6 @@ pub async fn stream_tenant_timelines_generic<'a>( }) } -pub(crate) fn stream_listing<'a>( - s3_client: &'a Client, - target: &'a S3Target, -) -> impl Stream> + 'a { - try_stream! { - let mut continuation_token = None; - loop { - let fetch_response = - list_objects_with_retries(s3_client, target, continuation_token.clone()).await?; - - if target.delimiter.is_empty() { - for object_key in fetch_response.contents().iter().filter_map(|object| object.key()) - { - let object_id = ObjectIdentifier::builder().key(object_key).build()?; - yield object_id; - } - } else { - for prefix in fetch_response.common_prefixes().iter().filter_map(|p| p.prefix()) { - let object_id = ObjectIdentifier::builder().key(prefix).build()?; - yield object_id; - } - } - - match fetch_response.next_continuation_token { - Some(new_token) => continuation_token = Some(new_token), - None => break, - } - } - } -} - pub(crate) fn stream_listing_generic<'a>( remote_client: &'a GenericRemoteStorage, target: &'a S3Target, diff --git a/storage_scrubber/src/pageserver_physical_gc.rs b/storage_scrubber/src/pageserver_physical_gc.rs index 20d9bd6dd4..6828081128 100644 --- a/storage_scrubber/src/pageserver_physical_gc.rs +++ b/storage_scrubber/src/pageserver_physical_gc.rs @@ -1,11 +1,10 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::Arc; -use std::time::{Duration, SystemTime}; +use std::time::Duration; -use crate::checks::{list_timeline_blobs, BlobDataParseResult}; -use crate::metadata_stream::{stream_tenant_timelines, stream_tenants}; -use crate::{init_remote, BucketConfig, NodeKind, RootTarget, TenantShardTimelineId}; -use aws_sdk_s3::Client; +use crate::checks::{list_timeline_blobs_generic, BlobDataParseResult}; +use crate::metadata_stream::{stream_tenant_timelines_generic, stream_tenants_generic}; +use crate::{init_remote_generic, BucketConfig, NodeKind, RootTarget, TenantShardTimelineId}; use futures_util::{StreamExt, TryStreamExt}; use pageserver::tenant::remote_timeline_client::index::LayerFileMetadata; use pageserver::tenant::remote_timeline_client::{parse_remote_index_path, remote_layer_path}; @@ -13,10 +12,11 @@ use pageserver::tenant::storage_layer::LayerName; use pageserver::tenant::IndexPart; use pageserver_api::controller_api::TenantDescribeResponse; use pageserver_api::shard::{ShardIndex, TenantShardId}; -use remote_storage::RemotePath; +use remote_storage::{GenericRemoteStorage, ListingObject, RemotePath}; use reqwest::Method; use serde::Serialize; use storage_controller_client::control_api; +use tokio_util::sync::CancellationToken; use tracing::{info_span, Instrument}; use utils::generation::Generation; use utils::id::{TenantId, TenantTimelineId}; @@ -240,38 +240,13 @@ impl TenantRefAccumulator { } } -async fn is_old_enough( - s3_client: &Client, - bucket_config: &BucketConfig, - min_age: &Duration, - key: &str, - summary: &mut GcSummary, -) -> bool { +fn is_old_enough(min_age: &Duration, key: &ListingObject, summary: &mut GcSummary) -> bool { // Validation: we will only GC indices & layers after a time threshold (e.g. one week) so that during an incident // it is easier to read old data for analysis, and easier to roll back shard splits without having to un-delete any objects. - let age: Duration = match s3_client - .head_object() - .bucket(&bucket_config.bucket) - .key(key) - .send() - .await - { - Ok(response) => match response.last_modified { - None => { - tracing::warn!("Missing last_modified"); - summary.remote_storage_errors += 1; - return false; - } - Some(last_modified) => match SystemTime::try_from(last_modified).map(|t| t.elapsed()) { - Ok(Ok(e)) => e, - Err(_) | Ok(Err(_)) => { - tracing::warn!("Bad last_modified time: {last_modified:?}"); - return false; - } - }, - }, - Err(e) => { - tracing::warn!("Failed to HEAD {key}: {e}"); + let age = match key.last_modified.elapsed() { + Ok(e) => e, + Err(_) => { + tracing::warn!("Bad last_modified time: {:?}", key.last_modified); summary.remote_storage_errors += 1; return false; } @@ -289,17 +264,30 @@ async fn is_old_enough( old_enough } +/// Same as [`is_old_enough`], but doesn't require a [`ListingObject`] passed to it. +async fn check_is_old_enough( + remote_client: &GenericRemoteStorage, + key: &RemotePath, + min_age: &Duration, + summary: &mut GcSummary, +) -> Option { + let listing_object = remote_client + .head_object(key, &CancellationToken::new()) + .await + .ok()?; + Some(is_old_enough(min_age, &listing_object, summary)) +} + async fn maybe_delete_index( - s3_client: &Client, - bucket_config: &BucketConfig, + remote_client: &GenericRemoteStorage, min_age: &Duration, latest_gen: Generation, - key: &str, + obj: &ListingObject, mode: GcMode, summary: &mut GcSummary, ) { // Validation: we will only delete things that parse cleanly - let basename = key.rsplit_once('/').unwrap().1; + let basename = obj.key.get_path().file_name().unwrap(); let candidate_generation = match parse_remote_index_path(RemotePath::from_string(basename).unwrap()) { Some(g) => g, @@ -328,7 +316,7 @@ async fn maybe_delete_index( return; } - if !is_old_enough(s3_client, bucket_config, min_age, key, summary).await { + if !is_old_enough(min_age, obj, summary) { return; } @@ -338,11 +326,8 @@ async fn maybe_delete_index( } // All validations passed: erase the object - match s3_client - .delete_object() - .bucket(&bucket_config.bucket) - .key(key) - .send() + match remote_client + .delete(&obj.key, &CancellationToken::new()) .await { Ok(_) => { @@ -358,8 +343,7 @@ async fn maybe_delete_index( #[allow(clippy::too_many_arguments)] async fn gc_ancestor( - s3_client: &Client, - bucket_config: &BucketConfig, + remote_client: &GenericRemoteStorage, root_target: &RootTarget, min_age: &Duration, ancestor: TenantShardId, @@ -368,7 +352,7 @@ async fn gc_ancestor( summary: &mut GcSummary, ) -> anyhow::Result<()> { // Scan timelines in the ancestor - let timelines = stream_tenant_timelines(s3_client, root_target, ancestor).await?; + let timelines = stream_tenant_timelines_generic(remote_client, root_target, ancestor).await?; let mut timelines = std::pin::pin!(timelines); // Build a list of keys to retain @@ -376,7 +360,7 @@ async fn gc_ancestor( while let Some(ttid) = timelines.next().await { let ttid = ttid?; - let data = list_timeline_blobs(s3_client, ttid, root_target).await?; + let data = list_timeline_blobs_generic(remote_client, ttid, root_target).await?; let s3_layers = match data.blob_data { BlobDataParseResult::Parsed { @@ -427,7 +411,8 @@ async fn gc_ancestor( // We apply a time threshold to GCing objects that are un-referenced: this preserves our ability // to roll back a shard split if we have to, by avoiding deleting ancestor layers right away - if !is_old_enough(s3_client, bucket_config, min_age, &key, summary).await { + let path = RemotePath::from_string(key.strip_prefix("/").unwrap_or(&key)).unwrap(); + if check_is_old_enough(remote_client, &path, min_age, summary).await != Some(true) { continue; } @@ -437,13 +422,7 @@ async fn gc_ancestor( } // All validations passed: erase the object - match s3_client - .delete_object() - .bucket(&bucket_config.bucket) - .key(&key) - .send() - .await - { + match remote_client.delete(&path, &CancellationToken::new()).await { Ok(_) => { tracing::info!("Successfully deleted unreferenced ancestor layer {key}"); summary.ancestor_layers_deleted += 1; @@ -477,10 +456,11 @@ pub async fn pageserver_physical_gc( min_age: Duration, mode: GcMode, ) -> anyhow::Result { - let (s3_client, target) = init_remote(bucket_config.clone(), NodeKind::Pageserver).await?; + let (remote_client, target) = + init_remote_generic(bucket_config.clone(), NodeKind::Pageserver).await?; let tenants = if tenant_shard_ids.is_empty() { - futures::future::Either::Left(stream_tenants(&s3_client, &target)) + futures::future::Either::Left(stream_tenants_generic(&remote_client, &target)) } else { futures::future::Either::Right(futures::stream::iter(tenant_shard_ids.into_iter().map(Ok))) }; @@ -493,14 +473,13 @@ pub async fn pageserver_physical_gc( let accumulator = Arc::new(std::sync::Mutex::new(TenantRefAccumulator::default())); // Generate a stream of TenantTimelineId - let timelines = tenants.map_ok(|t| stream_tenant_timelines(&s3_client, &target, t)); + let timelines = tenants.map_ok(|t| stream_tenant_timelines_generic(&remote_client, &target, t)); let timelines = timelines.try_buffered(CONCURRENCY); let timelines = timelines.try_flatten(); // Generate a stream of S3TimelineBlobData async fn gc_timeline( - s3_client: &Client, - bucket_config: &BucketConfig, + remote_client: &GenericRemoteStorage, min_age: &Duration, target: &RootTarget, mode: GcMode, @@ -508,7 +487,7 @@ pub async fn pageserver_physical_gc( accumulator: &Arc>, ) -> anyhow::Result { let mut summary = GcSummary::default(); - let data = list_timeline_blobs(s3_client, ttid, target).await?; + let data = list_timeline_blobs_generic(remote_client, ttid, target).await?; let (index_part, latest_gen, candidates) = match &data.blob_data { BlobDataParseResult::Parsed { @@ -533,17 +512,9 @@ pub async fn pageserver_physical_gc( accumulator.lock().unwrap().update(ttid, index_part); for key in candidates { - maybe_delete_index( - s3_client, - bucket_config, - min_age, - latest_gen, - &key, - mode, - &mut summary, - ) - .instrument(info_span!("maybe_delete_index", %ttid, ?latest_gen, key)) - .await; + maybe_delete_index(remote_client, min_age, latest_gen, &key, mode, &mut summary) + .instrument(info_span!("maybe_delete_index", %ttid, ?latest_gen, %key.key)) + .await; } Ok(summary) @@ -554,15 +525,7 @@ pub async fn pageserver_physical_gc( // Drain futures for per-shard GC, populating accumulator as a side effect { let timelines = timelines.map_ok(|ttid| { - gc_timeline( - &s3_client, - bucket_config, - &min_age, - &target, - mode, - ttid, - &accumulator, - ) + gc_timeline(&remote_client, &min_age, &target, mode, ttid, &accumulator) }); let mut timelines = std::pin::pin!(timelines.try_buffered(CONCURRENCY)); @@ -586,8 +549,7 @@ pub async fn pageserver_physical_gc( for ancestor_shard in ancestor_shards { gc_ancestor( - &s3_client, - bucket_config, + &remote_client, &target, &min_age, ancestor_shard, diff --git a/storage_scrubber/src/scan_pageserver_metadata.rs b/storage_scrubber/src/scan_pageserver_metadata.rs index 2409b7b132..e89e97ccb6 100644 --- a/storage_scrubber/src/scan_pageserver_metadata.rs +++ b/storage_scrubber/src/scan_pageserver_metadata.rs @@ -1,16 +1,16 @@ use std::collections::{HashMap, HashSet}; use crate::checks::{ - branch_cleanup_and_check_errors, list_timeline_blobs, BlobDataParseResult, S3TimelineBlobData, - TenantObjectListing, TimelineAnalysis, + branch_cleanup_and_check_errors, list_timeline_blobs_generic, BlobDataParseResult, + RemoteTimelineBlobData, TenantObjectListing, TimelineAnalysis, }; -use crate::metadata_stream::{stream_tenant_timelines, stream_tenants}; -use crate::{init_remote, BucketConfig, NodeKind, RootTarget, TenantShardTimelineId}; -use aws_sdk_s3::Client; +use crate::metadata_stream::{stream_tenant_timelines_generic, stream_tenants_generic}; +use crate::{init_remote_generic, BucketConfig, NodeKind, RootTarget, TenantShardTimelineId}; use futures_util::{StreamExt, TryStreamExt}; use pageserver::tenant::remote_timeline_client::remote_layer_path; use pageserver_api::controller_api::MetadataHealthUpdateRequest; use pageserver_api::shard::TenantShardId; +use remote_storage::GenericRemoteStorage; use serde::Serialize; use utils::id::TenantId; use utils::shard::ShardCount; @@ -36,7 +36,7 @@ impl MetadataSummary { Self::default() } - fn update_data(&mut self, data: &S3TimelineBlobData) { + fn update_data(&mut self, data: &RemoteTimelineBlobData) { self.timeline_shard_count += 1; if let BlobDataParseResult::Parsed { index_part, @@ -120,10 +120,10 @@ pub async fn scan_pageserver_metadata( bucket_config: BucketConfig, tenant_ids: Vec, ) -> anyhow::Result { - let (s3_client, target) = init_remote(bucket_config, NodeKind::Pageserver).await?; + let (remote_client, target) = init_remote_generic(bucket_config, NodeKind::Pageserver).await?; let tenants = if tenant_ids.is_empty() { - futures::future::Either::Left(stream_tenants(&s3_client, &target)) + futures::future::Either::Left(stream_tenants_generic(&remote_client, &target)) } else { futures::future::Either::Right(futures::stream::iter(tenant_ids.into_iter().map(Ok))) }; @@ -133,20 +133,20 @@ pub async fn scan_pageserver_metadata( const CONCURRENCY: usize = 32; // Generate a stream of TenantTimelineId - let timelines = tenants.map_ok(|t| stream_tenant_timelines(&s3_client, &target, t)); + let timelines = tenants.map_ok(|t| stream_tenant_timelines_generic(&remote_client, &target, t)); let timelines = timelines.try_buffered(CONCURRENCY); let timelines = timelines.try_flatten(); // Generate a stream of S3TimelineBlobData async fn report_on_timeline( - s3_client: &Client, + remote_client: &GenericRemoteStorage, target: &RootTarget, ttid: TenantShardTimelineId, - ) -> anyhow::Result<(TenantShardTimelineId, S3TimelineBlobData)> { - let data = list_timeline_blobs(s3_client, ttid, target).await?; + ) -> anyhow::Result<(TenantShardTimelineId, RemoteTimelineBlobData)> { + let data = list_timeline_blobs_generic(remote_client, ttid, target).await?; Ok((ttid, data)) } - let timelines = timelines.map_ok(|ttid| report_on_timeline(&s3_client, &target, ttid)); + let timelines = timelines.map_ok(|ttid| report_on_timeline(&remote_client, &target, ttid)); let mut timelines = std::pin::pin!(timelines.try_buffered(CONCURRENCY)); // We must gather all the TenantShardTimelineId->S3TimelineBlobData for each tenant, because different @@ -157,12 +157,11 @@ pub async fn scan_pageserver_metadata( let mut tenant_timeline_results = Vec::new(); async fn analyze_tenant( - s3_client: &Client, - target: &RootTarget, + remote_client: &GenericRemoteStorage, tenant_id: TenantId, summary: &mut MetadataSummary, mut tenant_objects: TenantObjectListing, - timelines: Vec<(TenantShardTimelineId, S3TimelineBlobData)>, + timelines: Vec<(TenantShardTimelineId, RemoteTimelineBlobData)>, highest_shard_count: ShardCount, ) { summary.tenant_count += 1; @@ -191,8 +190,7 @@ pub async fn scan_pageserver_metadata( // Apply checks to this timeline shard's metadata, and in the process update `tenant_objects` // reference counts for layers across the tenant. let analysis = branch_cleanup_and_check_errors( - s3_client, - target, + remote_client, &ttid, &mut tenant_objects, None, @@ -273,8 +271,7 @@ pub async fn scan_pageserver_metadata( let tenant_objects = std::mem::take(&mut tenant_objects); let timelines = std::mem::take(&mut tenant_timeline_results); analyze_tenant( - &s3_client, - &target, + &remote_client, prev_tenant_id, &mut summary, tenant_objects, @@ -311,8 +308,7 @@ pub async fn scan_pageserver_metadata( if !tenant_timeline_results.is_empty() { analyze_tenant( - &s3_client, - &target, + &remote_client, tenant_id.expect("Must be set if results are present"), &mut summary, tenant_objects, diff --git a/storage_scrubber/src/scan_safekeeper_metadata.rs b/storage_scrubber/src/scan_safekeeper_metadata.rs index 08a4541c5c..f20fa27d13 100644 --- a/storage_scrubber/src/scan_safekeeper_metadata.rs +++ b/storage_scrubber/src/scan_safekeeper_metadata.rs @@ -188,6 +188,11 @@ async fn check_timeline( // we need files, so unset it. timeline_dir_target.delimiter = String::new(); + let prefix_str = &timeline_dir_target + .prefix_in_bucket + .strip_prefix("/") + .unwrap_or(&timeline_dir_target.prefix_in_bucket); + let mut stream = std::pin::pin!(stream_listing_generic(remote_client, &timeline_dir_target)); while let Some(obj) = stream.next().await { let (key, _obj) = obj?; @@ -195,7 +200,7 @@ async fn check_timeline( let seg_name = key .get_path() .as_str() - .strip_prefix(&timeline_dir_target.prefix_in_bucket) + .strip_prefix(prefix_str) .expect("failed to extract segment name"); expected_segfiles.remove(seg_name); } diff --git a/storage_scrubber/src/tenant_snapshot.rs b/storage_scrubber/src/tenant_snapshot.rs index 1866e6ec80..fc3a973922 100644 --- a/storage_scrubber/src/tenant_snapshot.rs +++ b/storage_scrubber/src/tenant_snapshot.rs @@ -1,10 +1,11 @@ use std::collections::HashMap; use std::sync::Arc; -use crate::checks::{list_timeline_blobs, BlobDataParseResult, S3TimelineBlobData}; -use crate::metadata_stream::{stream_tenant_shards, stream_tenant_timelines}; +use crate::checks::{list_timeline_blobs_generic, BlobDataParseResult, RemoteTimelineBlobData}; +use crate::metadata_stream::{stream_tenant_shards, stream_tenant_timelines_generic}; use crate::{ - download_object_to_file, init_remote, BucketConfig, NodeKind, RootTarget, TenantShardTimelineId, + download_object_to_file, init_remote, init_remote_generic, BucketConfig, NodeKind, RootTarget, + TenantShardTimelineId, }; use anyhow::Context; use async_stream::stream; @@ -15,6 +16,7 @@ use pageserver::tenant::remote_timeline_client::index::LayerFileMetadata; use pageserver::tenant::storage_layer::LayerName; use pageserver::tenant::IndexPart; use pageserver_api::shard::TenantShardId; +use remote_storage::GenericRemoteStorage; use utils::generation::Generation; use utils::id::TenantId; @@ -215,11 +217,11 @@ impl SnapshotDownloader { } pub async fn download(&self) -> anyhow::Result<()> { - let (s3_client, target) = - init_remote(self.bucket_config.clone(), NodeKind::Pageserver).await?; + let (remote_client, target) = + init_remote_generic(self.bucket_config.clone(), NodeKind::Pageserver).await?; // Generate a stream of TenantShardId - let shards = stream_tenant_shards(&s3_client, &target, self.tenant_id).await?; + let shards = stream_tenant_shards(&remote_client, &target, self.tenant_id).await?; let shards: Vec = shards.try_collect().await?; // Only read from shards that have the highest count: avoids redundantly downloading @@ -237,18 +239,19 @@ impl SnapshotDownloader { for shard in shards.into_iter().filter(|s| s.shard_count == shard_count) { // Generate a stream of TenantTimelineId - let timelines = stream_tenant_timelines(&s3_client, &self.s3_root, shard).await?; + let timelines = stream_tenant_timelines_generic(&remote_client, &target, shard).await?; // Generate a stream of S3TimelineBlobData async fn load_timeline_index( - s3_client: &Client, + remote_client: &GenericRemoteStorage, target: &RootTarget, ttid: TenantShardTimelineId, - ) -> anyhow::Result<(TenantShardTimelineId, S3TimelineBlobData)> { - let data = list_timeline_blobs(s3_client, ttid, target).await?; + ) -> anyhow::Result<(TenantShardTimelineId, RemoteTimelineBlobData)> { + let data = list_timeline_blobs_generic(remote_client, ttid, target).await?; Ok((ttid, data)) } - let timelines = timelines.map_ok(|ttid| load_timeline_index(&s3_client, &target, ttid)); + let timelines = + timelines.map_ok(|ttid| load_timeline_index(&remote_client, &target, ttid)); let mut timelines = std::pin::pin!(timelines.try_buffered(8)); while let Some(i) = timelines.next().await { @@ -278,7 +281,7 @@ impl SnapshotDownloader { for (ttid, layers) in ancestor_layers.into_iter() { tracing::info!( - "Downloading {} layers from ancvestor timeline {ttid}...", + "Downloading {} layers from ancestor timeline {ttid}...", layers.len() ); diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index ba6fbc003a..9aa275d343 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -4643,6 +4643,7 @@ class StorageScrubber: ] args = base_args + args + log.info(f"Invoking scrubber command {args} with env: {env}") (output_path, stdout, status_code) = subprocess_capture( self.log_dir, args, From 6949b45e1795816507f5025a474e15d718e07456 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Mon, 19 Aug 2024 17:44:10 +0200 Subject: [PATCH 6/6] Update aws -> infra for repo rename (#8755) See slack thread: https://neondb.slack.com/archives/C039YKBRZB4/p1722501766006179 --- .github/workflows/build_and_test.yml | 10 +++++----- docs/rfcs/033-storage-controller-drain-and-fill.md | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index ee6d3ba005..92fff4ffbc 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -985,10 +985,10 @@ jobs: GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }} run: | if [[ "$GITHUB_REF_NAME" == "main" ]]; then - gh workflow --repo neondatabase/aws run deploy-dev.yml --ref main -f branch=main -f dockerTag=${{needs.tag.outputs.build-tag}} -f deployPreprodRegion=false + gh workflow --repo neondatabase/infra run deploy-dev.yml --ref main -f branch=main -f dockerTag=${{needs.tag.outputs.build-tag}} -f deployPreprodRegion=false gh workflow --repo neondatabase/azure run deploy.yml -f dockerTag=${{needs.tag.outputs.build-tag}} elif [[ "$GITHUB_REF_NAME" == "release" ]]; then - gh workflow --repo neondatabase/aws run deploy-dev.yml --ref main \ + gh workflow --repo neondatabase/infra run deploy-dev.yml --ref main \ -f deployPgSniRouter=false \ -f deployProxy=false \ -f deployStorage=true \ @@ -998,14 +998,14 @@ jobs: -f dockerTag=${{needs.tag.outputs.build-tag}} \ -f deployPreprodRegion=true - gh workflow --repo neondatabase/aws run deploy-prod.yml --ref main \ + gh workflow --repo neondatabase/infra run deploy-prod.yml --ref main \ -f deployStorage=true \ -f deployStorageBroker=true \ -f deployStorageController=true \ -f branch=main \ -f dockerTag=${{needs.tag.outputs.build-tag}} elif [[ "$GITHUB_REF_NAME" == "release-proxy" ]]; then - gh workflow --repo neondatabase/aws run deploy-dev.yml --ref main \ + gh workflow --repo neondatabase/infra run deploy-dev.yml --ref main \ -f deployPgSniRouter=true \ -f deployProxy=true \ -f deployStorage=false \ @@ -1015,7 +1015,7 @@ jobs: -f dockerTag=${{needs.tag.outputs.build-tag}} \ -f deployPreprodRegion=true - gh workflow --repo neondatabase/aws run deploy-proxy-prod.yml --ref main \ + gh workflow --repo neondatabase/infra run deploy-proxy-prod.yml --ref main \ -f deployPgSniRouter=true \ -f deployProxy=true \ -f branch=main \ diff --git a/docs/rfcs/033-storage-controller-drain-and-fill.md b/docs/rfcs/033-storage-controller-drain-and-fill.md index 77c84cd2a5..733f7c0bd8 100644 --- a/docs/rfcs/033-storage-controller-drain-and-fill.md +++ b/docs/rfcs/033-storage-controller-drain-and-fill.md @@ -14,7 +14,7 @@ picked tenant (which requested on-demand activation) for around 30 seconds during the restart at 2024-04-03 16:37 UTC. Note that lots of shutdowns on loaded pageservers do not finish within the -[10 second systemd enforced timeout](https://github.com/neondatabase/aws/blob/0a5280b383e43c063d43cbf87fa026543f6d6ad4/.github/ansible/systemd/pageserver.service#L16). This means we are shutting down without flushing ephemeral layers +[10 second systemd enforced timeout](https://github.com/neondatabase/infra/blob/0a5280b383e43c063d43cbf87fa026543f6d6ad4/.github/ansible/systemd/pageserver.service#L16). This means we are shutting down without flushing ephemeral layers and have to reingest data in order to serve requests after restarting, potentially making first request latencies worse. This problem is not yet very acutely felt in storage controller managed pageservers since