From 516ac0591e762142ca0ce85f212192c5af59a097 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 2 Sep 2024 12:36:57 +0100 Subject: [PATCH 01/45] storage controller: eliminate ensure_attached (#8875) ## Problem This is a followup to #8783 - The old blocking ensure_attached function had been retained to handle the case where a shard had a None generation_pageserver, but this wasn't really necessary. - There was a subtle `.1` in the code where a struct would have been clearer Closes #8819 ## Summary of changes - Add ShardGenerationState to represent the results of peek_generation - Instead of calling ensure_attached when a tenant has a non-attached shard, check the shard's policy and return 409 if it isn't Attached, else return 503 if the shard's policy is attached but it hasn't been reconciled yet (i.e. has a None generation_pageserver) --- storage_controller/src/persistence.rs | 22 +++-- storage_controller/src/service.rs | 124 +++++++++----------------- 2 files changed, 58 insertions(+), 88 deletions(-) diff --git a/storage_controller/src/persistence.rs b/storage_controller/src/persistence.rs index a842079ce7..6e1c2016ff 100644 --- a/storage_controller/src/persistence.rs +++ b/storage_controller/src/persistence.rs @@ -122,6 +122,13 @@ pub(crate) enum TenantFilter { Shard(TenantShardId), } +/// Represents the results of looking up generation+pageserver for the shards of a tenant +pub(crate) struct ShardGenerationState { + pub(crate) tenant_shard_id: TenantShardId, + pub(crate) generation: Option, + pub(crate) generation_pageserver: Option, +} + impl Persistence { // The default postgres connection limit is 100. We use up to 99, to leave one free for a human admin under // normal circumstances. This assumes we have exclusive use of the database cluster to which we connect. @@ -540,7 +547,7 @@ impl Persistence { pub(crate) async fn peek_generations( &self, filter_tenant_id: TenantId, - ) -> Result, Option)>, DatabaseError> { + ) -> Result, DatabaseError> { use crate::schema::tenant_shards::dsl::*; let rows = self .with_measured_conn(DatabaseOperation::PeekGenerations, move |conn| { @@ -555,13 +562,12 @@ impl Persistence { Ok(rows .into_iter() - .map(|p| { - ( - p.get_tenant_shard_id() - .expect("Corrupt tenant shard id in database"), - p.generation.map(|g| Generation::new(g as u32)), - p.generation_pageserver.map(|n| NodeId(n as u64)), - ) + .map(|p| ShardGenerationState { + tenant_shard_id: p + .get_tenant_shard_id() + .expect("Corrupt tenant shard id in database"), + generation: p.generation.map(|g| Generation::new(g as u32)), + generation_pageserver: p.generation_pageserver.map(|n| NodeId(n as u64)), }) .collect()) } diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 1f221a9b45..78627953d0 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -22,7 +22,7 @@ use crate::{ peer_client::GlobalObservedState, persistence::{ AbortShardSplitStatus, ControllerPersistence, DatabaseResult, MetadataHealthPersistence, - TenantFilter, + ShardGenerationState, TenantFilter, }, reconciler::{ReconcileError, ReconcileUnits, ReconcilerConfig, ReconcilerConfigBuilder}, scheduler::{MaySchedule, ScheduleContext, ScheduleMode}, @@ -3106,20 +3106,44 @@ impl Service { // will still be the latest when we're done: we will check generations again at the end of // this function to handle that. let generations = self.persistence.peek_generations(tenant_id).await?; - let generations = if generations.iter().any(|i| i.1.is_none()) { - // One or more shards is not attached to anything: maybe this is a new tenant? Wait for - // it to reconcile. - self.ensure_attached_wait(tenant_id).await?; - self.persistence.peek_generations(tenant_id).await? - } else { - generations - }; + + if generations + .iter() + .any(|i| i.generation.is_none() || i.generation_pageserver.is_none()) + { + // One or more shards has not been attached to a pageserver. Check if this is because it's configured + // to be detached (409: caller should give up), or because it's meant to be attached but isn't yet (503: caller should retry) + let locked = self.inner.read().unwrap(); + for (shard_id, shard) in + locked.tenants.range(TenantShardId::tenant_range(tenant_id)) + { + match shard.policy { + PlacementPolicy::Attached(_) => { + // This shard is meant to be attached: the caller is not wrong to try and + // use this function, but we can't service the request right now. + } + PlacementPolicy::Secondary | PlacementPolicy::Detached => { + return Err(ApiError::Conflict(format!( + "Shard {shard_id} tenant has policy {:?}", + shard.policy + ))); + } + } + } + + return Err(ApiError::ResourceUnavailable( + "One or more shards in tenant is not yet attached".into(), + )); + } let locked = self.inner.read().unwrap(); - for (tenant_shard_id, generation, generation_pageserver) in generations { - let node_id = generation_pageserver.ok_or(ApiError::Conflict( - "Tenant not currently attached".to_string(), - ))?; + for ShardGenerationState { + tenant_shard_id, + generation, + generation_pageserver, + } in generations + { + let node_id = generation_pageserver.expect("We checked for None above"); let node = locked .nodes .get(&node_id) @@ -3141,7 +3165,13 @@ impl Service { let latest_generations = self.persistence.peek_generations(tenant_id).await?; if latest_generations .into_iter() - .map(|g| (g.0, g.1)) + .map( + |ShardGenerationState { + tenant_shard_id, + generation, + generation_pageserver: _, + }| (tenant_shard_id, generation), + ) .collect::>() != target_gens .into_iter() @@ -5280,72 +5310,6 @@ impl Service { )) } - /// Helper for methods that will try and call pageserver APIs for - /// a tenant, such as timeline CRUD: they cannot proceed unless the tenant - /// is attached somewhere. - fn ensure_attached_schedule( - &self, - mut locked: std::sync::RwLockWriteGuard<'_, ServiceState>, - tenant_id: TenantId, - ) -> Result, anyhow::Error> { - let mut waiters = Vec::new(); - let (nodes, tenants, scheduler) = locked.parts_mut(); - - let mut schedule_context = ScheduleContext::default(); - for (tenant_shard_id, shard) in tenants.range_mut(TenantShardId::tenant_range(tenant_id)) { - shard.schedule(scheduler, &mut schedule_context)?; - - // The shard's policies may not result in an attached location being scheduled: this - // is an error because our caller needs it attached somewhere. - if shard.intent.get_attached().is_none() { - return Err(anyhow::anyhow!( - "Tenant {tenant_id} not scheduled to be attached" - )); - }; - - if shard.stably_attached().is_some() { - // We do not require the shard to be totally up to date on reconciliation: we just require - // that it has been attached on the intended node. Other dirty state such as unattached secondary - // locations, or compute hook notifications can be ignored. - continue; - } - - if let Some(waiter) = self.maybe_reconcile_shard(shard, nodes) { - tracing::info!("Waiting for shard {tenant_shard_id} to reconcile, in order to ensure it is attached"); - waiters.push(waiter); - } - } - Ok(waiters) - } - - async fn ensure_attached_wait(&self, tenant_id: TenantId) -> Result<(), ApiError> { - let ensure_waiters = { - let locked = self.inner.write().unwrap(); - - // Check if the tenant is splitting: in this case, even if it is attached, - // we must act as if it is not: this blocks e.g. timeline creation/deletion - // operations during the split. - for (_shard_id, shard) in locked.tenants.range(TenantShardId::tenant_range(tenant_id)) { - if !matches!(shard.splitting, SplitState::Idle) { - return Err(ApiError::ResourceUnavailable( - "Tenant shards are currently splitting".into(), - )); - } - } - - self.ensure_attached_schedule(locked, tenant_id) - .map_err(ApiError::InternalServerError)? - }; - - let deadline = Instant::now().checked_add(Duration::from_secs(5)).unwrap(); - for waiter in ensure_waiters { - let timeout = deadline.duration_since(Instant::now()); - waiter.wait_timeout(timeout).await?; - } - - Ok(()) - } - /// Like [`Self::maybe_configured_reconcile_shard`], but uses the default reconciler /// configuration fn maybe_reconcile_shard( From 9746b6ea312a15b2d607d5a124ca3899ec953d06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Mon, 2 Sep 2024 13:51:45 +0200 Subject: [PATCH 02/45] Implement archival_config timeline endpoint in the storage controller (#8680) Implement the timeline specific `archival_config` endpoint also in the storage controller. It's mostly a copy-paste of the detach handler: the task is the same: do the same operation on all shards. Part of #8088. --- pageserver/client/src/mgmt_api.rs | 18 +++++ storage_controller/src/http.rs | 30 ++++++- storage_controller/src/pageserver_client.rs | 20 ++++- storage_controller/src/service.rs | 73 ++++++++++++++++- test_runner/regress/test_timeline_archive.py | 83 +++++++++----------- 5 files changed, 174 insertions(+), 50 deletions(-) diff --git a/pageserver/client/src/mgmt_api.rs b/pageserver/client/src/mgmt_api.rs index 71d36f3113..737cb00835 100644 --- a/pageserver/client/src/mgmt_api.rs +++ b/pageserver/client/src/mgmt_api.rs @@ -419,6 +419,24 @@ impl Client { } } + pub async fn timeline_archival_config( + &self, + tenant_shard_id: TenantShardId, + timeline_id: TimelineId, + req: &TimelineArchivalConfigRequest, + ) -> Result<()> { + let uri = format!( + "{}/v1/tenant/{tenant_shard_id}/timeline/{timeline_id}/archival_config", + self.mgmt_api_endpoint + ); + + self.request(Method::POST, &uri, req) + .await? + .json() + .await + .map_err(Error::ReceiveBody) + } + pub async fn timeline_detach_ancestor( &self, tenant_shard_id: TenantShardId, diff --git a/storage_controller/src/http.rs b/storage_controller/src/http.rs index 207bd5a1e6..d3eb081be4 100644 --- a/storage_controller/src/http.rs +++ b/storage_controller/src/http.rs @@ -17,7 +17,7 @@ use pageserver_api::controller_api::{ }; use pageserver_api::models::{ TenantConfigRequest, TenantLocationConfigRequest, TenantShardSplitRequest, - TenantTimeTravelRequest, TimelineCreateRequest, + TenantTimeTravelRequest, TimelineArchivalConfigRequest, TimelineCreateRequest, }; use pageserver_api::shard::TenantShardId; use pageserver_client::mgmt_api; @@ -334,6 +334,24 @@ async fn handle_tenant_timeline_delete( .await } +async fn handle_tenant_timeline_archival_config( + service: Arc, + mut req: Request, +) -> Result, ApiError> { + let tenant_id: TenantId = parse_request_param(&req, "tenant_id")?; + check_permissions(&req, Scope::PageServerApi)?; + + let timeline_id: TimelineId = parse_request_param(&req, "timeline_id")?; + + let create_req = json_request::(&mut req).await?; + + service + .tenant_timeline_archival_config(tenant_id, timeline_id, create_req) + .await?; + + json_response(StatusCode::OK, ()) +} + async fn handle_tenant_timeline_detach_ancestor( service: Arc, req: Request, @@ -1160,6 +1178,16 @@ pub fn make_router( RequestName("v1_tenant_timeline"), ) }) + .post( + "/v1/tenant/:tenant_id/timeline/:timeline_id/archival_config", + |r| { + tenant_service_handler( + r, + handle_tenant_timeline_archival_config, + RequestName("v1_tenant_timeline_archival_config"), + ) + }, + ) .put( "/v1/tenant/:tenant_id/timeline/:timeline_id/detach_ancestor", |r| { diff --git a/storage_controller/src/pageserver_client.rs b/storage_controller/src/pageserver_client.rs index 8d64201cd9..20770ed703 100644 --- a/storage_controller/src/pageserver_client.rs +++ b/storage_controller/src/pageserver_client.rs @@ -2,8 +2,8 @@ use pageserver_api::{ models::{ detach_ancestor::AncestorDetached, LocationConfig, LocationConfigListResponse, PageserverUtilization, SecondaryProgress, TenantScanRemoteStorageResponse, - TenantShardSplitRequest, TenantShardSplitResponse, TimelineCreateRequest, TimelineInfo, - TopTenantShardsRequest, TopTenantShardsResponse, + TenantShardSplitRequest, TenantShardSplitResponse, TimelineArchivalConfigRequest, + TimelineCreateRequest, TimelineInfo, TopTenantShardsRequest, TopTenantShardsResponse, }, shard::TenantShardId, }; @@ -227,6 +227,22 @@ impl PageserverClient { ) } + pub(crate) async fn timeline_archival_config( + &self, + tenant_shard_id: TenantShardId, + timeline_id: TimelineId, + req: &TimelineArchivalConfigRequest, + ) -> Result<()> { + measured_request!( + "timeline_archival_config", + crate::metrics::Method::Post, + &self.node_id_label, + self.inner + .timeline_archival_config(tenant_shard_id, timeline_id, req) + .await + ) + } + pub(crate) async fn timeline_detach_ancestor( &self, tenant_shard_id: TenantShardId, diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 78627953d0..95821827e2 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -46,7 +46,10 @@ use pageserver_api::{ TenantDescribeResponseShard, TenantLocateResponse, TenantPolicyRequest, TenantShardMigrateRequest, TenantShardMigrateResponse, }, - models::{SecondaryProgress, TenantConfigRequest, TopTenantShardsRequest}, + models::{ + SecondaryProgress, TenantConfigRequest, TimelineArchivalConfigRequest, + TopTenantShardsRequest, + }, }; use reqwest::StatusCode; use tracing::{instrument, Instrument}; @@ -131,6 +134,7 @@ enum TenantOperations { TimelineCreate, TimelineDelete, AttachHook, + TimelineArchivalConfig, TimelineDetachAncestor, } @@ -2918,6 +2922,73 @@ impl Service { .await? } + pub(crate) async fn tenant_timeline_archival_config( + &self, + tenant_id: TenantId, + timeline_id: TimelineId, + req: TimelineArchivalConfigRequest, + ) -> Result<(), ApiError> { + tracing::info!( + "Setting archival config of timeline {tenant_id}/{timeline_id} to '{:?}'", + req.state + ); + + let _tenant_lock = trace_shared_lock( + &self.tenant_op_locks, + tenant_id, + TenantOperations::TimelineArchivalConfig, + ) + .await; + + self.tenant_remote_mutation(tenant_id, move |targets| async move { + if targets.is_empty() { + return Err(ApiError::NotFound( + anyhow::anyhow!("Tenant not found").into(), + )); + } + async fn config_one( + tenant_shard_id: TenantShardId, + timeline_id: TimelineId, + node: Node, + jwt: Option, + req: TimelineArchivalConfigRequest, + ) -> Result<(), ApiError> { + tracing::info!( + "Setting archival config of timeline on shard {tenant_shard_id}/{timeline_id}, attached to node {node}", + ); + + let client = PageserverClient::new(node.get_id(), node.base_url(), jwt.as_deref()); + + client + .timeline_archival_config(tenant_shard_id, timeline_id, &req) + .await + .map_err(|e| match e { + mgmt_api::Error::ApiError(StatusCode::PRECONDITION_FAILED, msg) => { + ApiError::PreconditionFailed(msg.into_boxed_str()) + } + _ => passthrough_api_error(&node, e), + }) + } + + // no shard needs to go first/last; the operation should be idempotent + // TODO: it would be great to ensure that all shards return the same error + let results = self + .tenant_for_shards(targets, |tenant_shard_id, node| { + futures::FutureExt::boxed(config_one( + tenant_shard_id, + timeline_id, + node, + self.config.jwt_token.clone(), + req.clone(), + )) + }) + .await?; + assert!(!results.is_empty(), "must have at least one result"); + + Ok(()) + }).await? + } + pub(crate) async fn tenant_timeline_detach_ancestor( &self, tenant_id: TenantId, diff --git a/test_runner/regress/test_timeline_archive.py b/test_runner/regress/test_timeline_archive.py index 7f158ad251..de43e51c9e 100644 --- a/test_runner/regress/test_timeline_archive.py +++ b/test_runner/regress/test_timeline_archive.py @@ -1,97 +1,90 @@ import pytest from fixtures.common_types import TenantId, TimelineArchivalState, TimelineId from fixtures.neon_fixtures import ( - NeonEnv, + NeonEnvBuilder, ) from fixtures.pageserver.http import PageserverApiException -def test_timeline_archive(neon_simple_env: NeonEnv): - env = neon_simple_env +@pytest.mark.parametrize("shard_count", [0, 4]) +def test_timeline_archive(neon_env_builder: NeonEnvBuilder, shard_count: int): + unsharded = shard_count == 0 + if unsharded: + env = neon_env_builder.init_start() + # If we run the unsharded version, talk to the pageserver directly + ps_http = env.pageserver.http_client() + else: + neon_env_builder.num_pageservers = shard_count + env = neon_env_builder.init_start(initial_tenant_shard_count=shard_count) + # If we run the unsharded version, talk to the storage controller + ps_http = env.storage_controller.pageserver_api() - env.pageserver.allowed_errors.extend( - [ - ".*Timeline .* was not found.*", - ".*timeline not found.*", - ".*Cannot archive timeline which has unarchived child timelines.*", - ".*Precondition failed: Requested tenant is missing.*", - ] - ) - - ps_http = env.pageserver.http_client() - - # first try to archive non existing timeline - # for existing tenant: + # first try to archive a non existing timeline for an existing tenant: invalid_timeline_id = TimelineId.generate() with pytest.raises(PageserverApiException, match="timeline not found") as exc: ps_http.timeline_archival_config( - tenant_id=env.initial_tenant, - timeline_id=invalid_timeline_id, + env.initial_tenant, + invalid_timeline_id, state=TimelineArchivalState.ARCHIVED, ) assert exc.value.status_code == 404 - # for non existing tenant: + # for a non existing tenant: invalid_tenant_id = TenantId.generate() with pytest.raises( PageserverApiException, - match=f"NotFound: tenant {invalid_tenant_id}", + match="NotFound: [tT]enant", ) as exc: ps_http.timeline_archival_config( - tenant_id=invalid_tenant_id, - timeline_id=invalid_timeline_id, + invalid_tenant_id, + invalid_timeline_id, state=TimelineArchivalState.ARCHIVED, ) assert exc.value.status_code == 404 - # construct pair of branches to validate that pageserver prohibits + # construct a pair of branches to validate that pageserver prohibits # archival of ancestor timelines when they have non-archived child branches - parent_timeline_id = env.neon_cli.create_branch("test_ancestor_branch_archive_parent", "empty") + parent_timeline_id = env.neon_cli.create_branch("test_ancestor_branch_archive_parent") leaf_timeline_id = env.neon_cli.create_branch( "test_ancestor_branch_archive_branch1", "test_ancestor_branch_archive_parent" ) - timeline_path = env.pageserver.timeline_dir(env.initial_tenant, parent_timeline_id) - with pytest.raises( PageserverApiException, match="Cannot archive timeline which has non-archived child timelines", ) as exc: - assert timeline_path.exists() - ps_http.timeline_archival_config( - tenant_id=env.initial_tenant, - timeline_id=parent_timeline_id, + env.initial_tenant, + parent_timeline_id, state=TimelineArchivalState.ARCHIVED, ) assert exc.value.status_code == 412 - # Test timeline_detail leaf_detail = ps_http.timeline_detail( - tenant_id=env.initial_tenant, + env.initial_tenant, timeline_id=leaf_timeline_id, ) assert leaf_detail["is_archived"] is False # Test that archiving the leaf timeline and then the parent works ps_http.timeline_archival_config( - tenant_id=env.initial_tenant, - timeline_id=leaf_timeline_id, + env.initial_tenant, + leaf_timeline_id, state=TimelineArchivalState.ARCHIVED, ) leaf_detail = ps_http.timeline_detail( - tenant_id=env.initial_tenant, - timeline_id=leaf_timeline_id, + env.initial_tenant, + leaf_timeline_id, ) assert leaf_detail["is_archived"] is True ps_http.timeline_archival_config( - tenant_id=env.initial_tenant, - timeline_id=parent_timeline_id, + env.initial_tenant, + parent_timeline_id, state=TimelineArchivalState.ARCHIVED, ) @@ -100,23 +93,21 @@ def test_timeline_archive(neon_simple_env: NeonEnv): PageserverApiException, match="ancestor is archived", ) as exc: - assert timeline_path.exists() - ps_http.timeline_archival_config( - tenant_id=env.initial_tenant, - timeline_id=leaf_timeline_id, + env.initial_tenant, + leaf_timeline_id, state=TimelineArchivalState.UNARCHIVED, ) # Unarchive works for the leaf if the parent gets unarchived first ps_http.timeline_archival_config( - tenant_id=env.initial_tenant, - timeline_id=parent_timeline_id, + env.initial_tenant, + parent_timeline_id, state=TimelineArchivalState.UNARCHIVED, ) ps_http.timeline_archival_config( - tenant_id=env.initial_tenant, - timeline_id=leaf_timeline_id, + env.initial_tenant, + leaf_timeline_id, state=TimelineArchivalState.UNARCHIVED, ) From 15e90cc427aad8f9ded4e0c13a283217631cbd07 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Mon, 2 Sep 2024 15:45:17 +0200 Subject: [PATCH 03/45] bottommost-compaction: remove dead code / rectify cfg!()s (#8884) part of https://github.com/neondatabase/neon/issues/8002 --- .../src/tenant/storage_layer/delta_layer.rs | 40 ------------------- .../src/tenant/storage_layer/image_layer.rs | 32 ++------------- pageserver/src/tenant/storage_layer/layer.rs | 31 -------------- .../src/tenant/storage_layer/layer/tests.rs | 2 +- .../src/tenant/storage_layer/split_writer.rs | 2 +- pageserver/src/tenant/timeline.rs | 1 - 6 files changed, 5 insertions(+), 103 deletions(-) diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 885eb13b29..b8e9a98149 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -225,9 +225,7 @@ pub struct DeltaLayerInner { file: VirtualFile, file_id: FileId, - #[allow(dead_code)] layer_key_range: Range, - #[allow(dead_code)] layer_lsn_range: Range, max_vectored_read_bytes: Option, @@ -882,44 +880,6 @@ impl DeltaLayerInner { Ok(()) } - /// Load all key-values in the delta layer, should be replaced by an iterator-based interface in the future. - pub(super) async fn load_key_values( - &self, - ctx: &RequestContext, - ) -> anyhow::Result> { - let block_reader = FileBlockReader::new(&self.file, self.file_id); - let index_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new( - self.index_start_blk, - self.index_root_blk, - block_reader, - ); - let mut result = Vec::new(); - let mut stream = - Box::pin(self.stream_index_forwards(index_reader, &[0; DELTA_KEY_SIZE], ctx)); - let block_reader = FileBlockReader::new(&self.file, self.file_id); - let cursor = block_reader.block_cursor(); - let mut buf = Vec::new(); - while let Some(item) = stream.next().await { - let (key, lsn, pos) = item?; - // TODO: dedup code with get_reconstruct_value - // TODO: ctx handling and sharding - cursor - .read_blob_into_buf(pos.pos(), &mut buf, ctx) - .await - .with_context(|| { - format!("Failed to read blob from virtual file {}", self.file.path) - })?; - let val = Value::des(&buf).with_context(|| { - format!( - "Failed to deserialize file blob from virtual file {}", - self.file.path - ) - })?; - result.push((key, lsn, val)); - } - Ok(result) - } - async fn plan_reads( keyspace: &KeySpace, lsn_range: Range, diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index 4c22541e02..4a095c564d 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -28,7 +28,7 @@ use crate::context::{PageContentKind, RequestContext, RequestContextBuilder}; use crate::page_cache::{self, FileId, PAGE_SZ}; use crate::repository::{Key, Value, KEY_SIZE}; use crate::tenant::blob_io::BlobWriter; -use crate::tenant::block_io::{BlockBuf, BlockReader, FileBlockReader}; +use crate::tenant::block_io::{BlockBuf, FileBlockReader}; use crate::tenant::disk_btree::{ DiskBtreeBuilder, DiskBtreeIterator, DiskBtreeReader, VisitDirection, }; @@ -453,33 +453,6 @@ impl ImageLayerInner { Ok(()) } - /// Load all key-values in the delta layer, should be replaced by an iterator-based interface in the future. - pub(super) async fn load_key_values( - &self, - ctx: &RequestContext, - ) -> anyhow::Result> { - let block_reader = FileBlockReader::new(&self.file, self.file_id); - let tree_reader = - DiskBtreeReader::new(self.index_start_blk, self.index_root_blk, &block_reader); - let mut result = Vec::new(); - let mut stream = Box::pin(tree_reader.into_stream(&[0; KEY_SIZE], ctx)); - let block_reader = FileBlockReader::new(&self.file, self.file_id); - let cursor = block_reader.block_cursor(); - while let Some(item) = stream.next().await { - // TODO: dedup code with get_reconstruct_value - let (raw_key, offset) = item?; - let key = Key::from_slice(&raw_key[..KEY_SIZE]); - // TODO: ctx handling and sharding - let blob = cursor - .read_blob(offset, ctx) - .await - .with_context(|| format!("failed to read value from offset {}", offset))?; - let value = Bytes::from(blob); - result.push((key, self.lsn, Value::Image(value))); - } - Ok(result) - } - /// Traverse the layer's index to build read operations on the overlap of the input keyspace /// and the keys in this layer. /// @@ -711,7 +684,7 @@ struct ImageLayerWriterInner { blob_writer: BlobWriter, tree: DiskBtreeBuilder, - #[cfg_attr(not(feature = "testing"), allow(dead_code))] + #[cfg(feature = "testing")] last_written_key: Key, } @@ -770,6 +743,7 @@ impl ImageLayerWriterInner { uncompressed_bytes_eligible: 0, uncompressed_bytes_chosen: 0, num_keys: 0, + #[cfg(feature = "testing")] last_written_key: Key::MIN, }; diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index 86a200ce28..56f5cc556d 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -14,7 +14,6 @@ use utils::sync::{gate, heavier_once_cell}; use crate::config::PageServerConf; use crate::context::{DownloadBehavior, RequestContext}; -use crate::repository::Key; use crate::span::debug_assert_current_span_has_tenant_and_timeline_id; use crate::task_mgr::TaskKind; use crate::tenant::timeline::{CompactionError, GetVectoredError}; @@ -334,23 +333,6 @@ impl Layer { }) } - /// Get all key/values in the layer. Should be replaced with an iterator-based API in the future. - #[allow(dead_code)] - pub(crate) async fn load_key_values( - &self, - ctx: &RequestContext, - ) -> anyhow::Result> { - let layer = self - .0 - .get_or_maybe_download(true, Some(ctx)) - .await - .map_err(|err| match err { - DownloadError::DownloadCancelled => GetVectoredError::Cancelled, - other => GetVectoredError::Other(anyhow::anyhow!(other)), - })?; - layer.load_key_values(&self.0, ctx).await - } - /// Download the layer if evicted. /// /// Will not error when the layer is already downloaded. @@ -1777,19 +1759,6 @@ impl DownloadedLayer { } } - async fn load_key_values( - &self, - owner: &Arc, - ctx: &RequestContext, - ) -> anyhow::Result> { - use LayerKind::*; - - match self.get(owner, ctx).await? { - Delta(d) => d.load_key_values(ctx).await, - Image(i) => i.load_key_values(ctx).await, - } - } - async fn dump(&self, owner: &Arc, ctx: &RequestContext) -> anyhow::Result<()> { use LayerKind::*; match self.get(owner, ctx).await? { diff --git a/pageserver/src/tenant/storage_layer/layer/tests.rs b/pageserver/src/tenant/storage_layer/layer/tests.rs index bffd2db800..0b9bde4f57 100644 --- a/pageserver/src/tenant/storage_layer/layer/tests.rs +++ b/pageserver/src/tenant/storage_layer/layer/tests.rs @@ -782,7 +782,7 @@ async fn eviction_cancellation_on_drop() { let mut writer = timeline.writer().await; writer .put( - Key::from_i128(5), + crate::repository::Key::from_i128(5), Lsn(0x20), &Value::Image(Bytes::from_static(b"this does not matter either")), &ctx, diff --git a/pageserver/src/tenant/storage_layer/split_writer.rs b/pageserver/src/tenant/storage_layer/split_writer.rs index df910b5ad9..e8deb0a1e5 100644 --- a/pageserver/src/tenant/storage_layer/split_writer.rs +++ b/pageserver/src/tenant/storage_layer/split_writer.rs @@ -353,7 +353,7 @@ impl SplitDeltaLayerWriter { Ok(generated_layers) } - #[allow(dead_code)] + #[cfg(test)] pub(crate) async fn finish( self, tline: &Arc, diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 35e0825bac..6eadf9a564 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -4537,7 +4537,6 @@ pub struct DeltaLayerTestDesc { #[cfg(test)] impl DeltaLayerTestDesc { - #[allow(dead_code)] pub fn new(lsn_range: Range, key_range: Range, data: Vec<(Key, Lsn, Value)>) -> Self { Self { lsn_range, From bf0531d10703e1f6cd92e29ca69a9bb68503121e Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Mon, 2 Sep 2024 16:10:10 +0200 Subject: [PATCH 04/45] fixup(#8839): `test_forward_compatibility` needs to allow lag warning as well (#8891) Found in https://neon-github-public-dev.s3.amazonaws.com/reports/pr-8885/10665614629/index.html#suites/0fbaeb107ef328d03993d44a1fb15690/ea10ba1c140fba1d --- test_runner/regress/test_compatibility.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test_runner/regress/test_compatibility.py b/test_runner/regress/test_compatibility.py index cd3f405a86..467e5b1734 100644 --- a/test_runner/regress/test_compatibility.py +++ b/test_runner/regress/test_compatibility.py @@ -149,6 +149,10 @@ def test_create_snapshot( ) +# check_neon_works does recovery from WAL => the compatibility snapshot's WAL is old => will log this warning +ingest_lag_log_line = ".*ingesting record with timestamp lagging more than wait_lsn_timeout.*" + + @check_ondisk_data_compatibility_if_enabled @pytest.mark.xdist_group("compatibility") @pytest.mark.order(after="test_create_snapshot") @@ -173,10 +177,6 @@ def test_backward_compatibility( try: neon_env_builder.num_safekeepers = 3 env = neon_env_builder.from_repo_dir(compatibility_snapshot_dir / "repo") - # check_neon_works does recovery from WAL => the compatibility snapshot's WAL is old => will log this warning - ingest_lag_log_line = ( - ".*ingesting record with timestamp lagging more than wait_lsn_timeout.*" - ) env.pageserver.allowed_errors.append(ingest_lag_log_line) neon_env_builder.start() @@ -246,6 +246,8 @@ def test_forward_compatibility( env = neon_env_builder.from_repo_dir( compatibility_snapshot_dir / "repo", ) + # there may be an arbitrary number of unrelated tests run between create_snapshot and here + env.pageserver.allowed_errors.append(ingest_lag_log_line) # not using env.pageserver.version because it was initialized before prev_pageserver_version_str = env.get_binary_version("pageserver") From 3b317cae071a7eb84247e616e15541868e292bd3 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Mon, 2 Sep 2024 17:09:26 +0200 Subject: [PATCH 05/45] page_cache/layer load: correctly classify layer summary block reads (#8885) Before this PR, we would classify layer summary block reads as "Unknown" content kind. image --- pageserver/src/context.rs | 2 ++ pageserver/src/tenant/storage_layer/layer.rs | 12 +++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pageserver/src/context.rs b/pageserver/src/context.rs index 0b07e07524..012cb8d96f 100644 --- a/pageserver/src/context.rs +++ b/pageserver/src/context.rs @@ -105,8 +105,10 @@ pub struct RequestContext { #[derive(Clone, Copy, PartialEq, Eq, Debug, enum_map::Enum, strum_macros::IntoStaticStr)] pub enum PageContentKind { Unknown, + DeltaLayerSummary, DeltaLayerBtreeNode, DeltaLayerValue, + ImageLayerSummary, ImageLayerBtreeNode, ImageLayerValue, InMemoryLayer, diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index 56f5cc556d..b15cd4da39 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -13,7 +13,7 @@ use utils::lsn::Lsn; use utils::sync::{gate, heavier_once_cell}; use crate::config::PageServerConf; -use crate::context::{DownloadBehavior, RequestContext}; +use crate::context::{DownloadBehavior, RequestContext, RequestContextBuilder}; use crate::span::debug_assert_current_span_has_tenant_and_timeline_id; use crate::task_mgr::TaskKind; use crate::tenant::timeline::{CompactionError, GetVectoredError}; @@ -1678,6 +1678,9 @@ impl DownloadedLayer { ); let res = if owner.desc.is_delta { + let ctx = RequestContextBuilder::extend(ctx) + .page_content_kind(crate::context::PageContentKind::DeltaLayerSummary) + .build(); let summary = Some(delta_layer::Summary::expected( owner.desc.tenant_shard_id.tenant_id, owner.desc.timeline_id, @@ -1688,11 +1691,14 @@ impl DownloadedLayer { &owner.path, summary, Some(owner.conf.max_vectored_read_bytes), - ctx, + &ctx, ) .await .map(LayerKind::Delta) } else { + let ctx = RequestContextBuilder::extend(ctx) + .page_content_kind(crate::context::PageContentKind::ImageLayerSummary) + .build(); let lsn = owner.desc.image_layer_lsn(); let summary = Some(image_layer::Summary::expected( owner.desc.tenant_shard_id.tenant_id, @@ -1705,7 +1711,7 @@ impl DownloadedLayer { lsn, summary, Some(owner.conf.max_vectored_read_bytes), - ctx, + &ctx, ) .await .map(LayerKind::Image) From b37da32c6f56f31f39661c9364a7a662df59dbbc Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Tue, 3 Sep 2024 10:05:24 +0200 Subject: [PATCH 06/45] pageserver: reuse idempotency keys across metrics sinks (#8876) ## Problem Metrics event idempotency keys differ across S3 and Vector. The events should be identical. Resolves #8605. ## Summary of changes Pre-generate the idempotency keys and pass the same set into both metrics sinks. Co-authored-by: John Spray --- pageserver/src/consumption_metrics.rs | 24 +++++++-- pageserver/src/consumption_metrics/upload.rs | 52 +++++++++++--------- 2 files changed, 47 insertions(+), 29 deletions(-) diff --git a/pageserver/src/consumption_metrics.rs b/pageserver/src/consumption_metrics.rs index f94d945d46..64a267e0e4 100644 --- a/pageserver/src/consumption_metrics.rs +++ b/pageserver/src/consumption_metrics.rs @@ -1,6 +1,8 @@ //! Periodically collect consumption metrics for all active tenants //! and push them to a HTTP endpoint. use crate::config::PageServerConf; +use crate::consumption_metrics::metrics::MetricsKey; +use crate::consumption_metrics::upload::KeyGen as _; use crate::context::{DownloadBehavior, RequestContext}; use crate::task_mgr::{self, TaskKind, BACKGROUND_RUNTIME}; use crate::tenant::size::CalculateSyntheticSizeError; @@ -8,6 +10,7 @@ use crate::tenant::tasks::BackgroundLoopKind; use crate::tenant::{mgr::TenantManager, LogicalSizeCalculationCause, Tenant}; use camino::Utf8PathBuf; use consumption_metrics::EventType; +use itertools::Itertools as _; use pageserver_api::models::TenantState; use remote_storage::{GenericRemoteStorage, RemoteStorageConfig}; use reqwest::Url; @@ -19,9 +22,8 @@ use tokio_util::sync::CancellationToken; use tracing::*; use utils::id::NodeId; -mod metrics; -use crate::consumption_metrics::metrics::MetricsKey; mod disk_cache; +mod metrics; mod upload; const DEFAULT_HTTP_REPORTING_TIMEOUT: Duration = Duration::from_secs(60); @@ -143,6 +145,12 @@ async fn collect_metrics( // these are point in time, with variable "now" let metrics = metrics::collect_all_metrics(&tenant_manager, &cached_metrics, &ctx).await; + // Pre-generate event idempotency keys, to reuse them across the bucket + // and HTTP sinks. + let idempotency_keys = std::iter::repeat_with(|| node_id.as_str().generate()) + .take(metrics.len()) + .collect_vec(); + let metrics = Arc::new(metrics); // why not race cancellation here? because we are one of the last tasks, and if we are @@ -161,8 +169,14 @@ async fn collect_metrics( } if let Some(bucket_client) = &bucket_client { - let res = - upload::upload_metrics_bucket(bucket_client, &cancel, &node_id, &metrics).await; + let res = upload::upload_metrics_bucket( + bucket_client, + &cancel, + &node_id, + &metrics, + &idempotency_keys, + ) + .await; if let Err(e) = res { tracing::error!("failed to upload to S3: {e:#}"); } @@ -174,9 +188,9 @@ async fn collect_metrics( &client, metric_collection_endpoint, &cancel, - &node_id, &metrics, &mut cached_metrics, + &idempotency_keys, ) .await; if let Err(e) = res { diff --git a/pageserver/src/consumption_metrics/upload.rs b/pageserver/src/consumption_metrics/upload.rs index 4e8283c3e4..0325ee403a 100644 --- a/pageserver/src/consumption_metrics/upload.rs +++ b/pageserver/src/consumption_metrics/upload.rs @@ -24,16 +24,16 @@ pub(super) async fn upload_metrics_http( client: &reqwest::Client, metric_collection_endpoint: &reqwest::Url, cancel: &CancellationToken, - node_id: &str, metrics: &[RawMetric], cached_metrics: &mut Cache, + idempotency_keys: &[IdempotencyKey<'_>], ) -> anyhow::Result<()> { let mut uploaded = 0; let mut failed = 0; let started_at = std::time::Instant::now(); - let mut iter = serialize_in_chunks(CHUNK_SIZE, metrics, node_id); + let mut iter = serialize_in_chunks(CHUNK_SIZE, metrics, idempotency_keys); while let Some(res) = iter.next() { let (chunk, body) = res?; @@ -87,6 +87,7 @@ pub(super) async fn upload_metrics_bucket( cancel: &CancellationToken, node_id: &str, metrics: &[RawMetric], + idempotency_keys: &[IdempotencyKey<'_>], ) -> anyhow::Result<()> { if metrics.is_empty() { // Skip uploads if we have no metrics, so that readers don't have to handle the edge case @@ -106,7 +107,7 @@ pub(super) async fn upload_metrics_bucket( // Serialize and write into compressed buffer let started_at = std::time::Instant::now(); - for res in serialize_in_chunks(CHUNK_SIZE, metrics, node_id) { + for res in serialize_in_chunks(CHUNK_SIZE, metrics, idempotency_keys) { let (_chunk, body) = res?; gzip_writer.write_all(&body).await?; } @@ -134,29 +135,31 @@ pub(super) async fn upload_metrics_bucket( Ok(()) } -// The return type is quite ugly, but we gain testability in isolation -fn serialize_in_chunks<'a, F>( +/// Serializes the input metrics as JSON in chunks of chunk_size. The provided +/// idempotency keys are injected into the corresponding metric events (reused +/// across different metrics sinks), and must have the same length as input. +fn serialize_in_chunks<'a>( chunk_size: usize, input: &'a [RawMetric], - factory: F, + idempotency_keys: &'a [IdempotencyKey<'a>], ) -> impl ExactSizeIterator> + 'a -where - F: KeyGen<'a> + 'a, { use bytes::BufMut; - struct Iter<'a, F> { + assert_eq!(input.len(), idempotency_keys.len()); + + struct Iter<'a> { inner: std::slice::Chunks<'a, RawMetric>, + idempotency_keys: std::slice::Iter<'a, IdempotencyKey<'a>>, chunk_size: usize, // write to a BytesMut so that we can cheaply clone the frozen Bytes for retries buffer: bytes::BytesMut, // chunk amount of events are reused to produce the serialized document scratch: Vec>, - factory: F, } - impl<'a, F: KeyGen<'a>> Iterator for Iter<'a, F> { + impl<'a> Iterator for Iter<'a> { type Item = Result<(&'a [RawMetric], bytes::Bytes), serde_json::Error>; fn next(&mut self) -> Option { @@ -167,17 +170,14 @@ where self.scratch.extend( chunk .iter() - .map(|raw_metric| raw_metric.as_event(&self.factory.generate())), + .zip(&mut self.idempotency_keys) + .map(|(raw_metric, key)| raw_metric.as_event(key)), ); } else { // next rounds: update_in_place to reuse allocations assert_eq!(self.scratch.len(), self.chunk_size); - self.scratch - .iter_mut() - .zip(chunk.iter()) - .for_each(|(slot, raw_metric)| { - raw_metric.update_in_place(slot, &self.factory.generate()) - }); + itertools::izip!(self.scratch.iter_mut(), chunk, &mut self.idempotency_keys) + .for_each(|(slot, raw_metric, key)| raw_metric.update_in_place(slot, key)); } let res = serde_json::to_writer( @@ -198,18 +198,19 @@ where } } - impl<'a, F: KeyGen<'a>> ExactSizeIterator for Iter<'a, F> {} + impl<'a> ExactSizeIterator for Iter<'a> {} let buffer = bytes::BytesMut::new(); let inner = input.chunks(chunk_size); + let idempotency_keys = idempotency_keys.iter(); let scratch = Vec::new(); Iter { inner, + idempotency_keys, chunk_size, buffer, scratch, - factory, } } @@ -268,7 +269,7 @@ impl RawMetricExt for RawMetric { } } -trait KeyGen<'a>: Copy { +pub(crate) trait KeyGen<'a> { fn generate(&self) -> IdempotencyKey<'a>; } @@ -389,7 +390,10 @@ mod tests { let examples = metric_samples(); assert!(examples.len() > 1); - let factory = FixedGen::new(Utc::now(), "1", 42); + let now = Utc::now(); + let idempotency_keys = (0..examples.len()) + .map(|i| FixedGen::new(now, "1", i as u16).generate()) + .collect::>(); // need to use Event here because serde_json::Value uses default hashmap, not linked // hashmap @@ -398,13 +402,13 @@ mod tests { events: Vec>, } - let correct = serialize_in_chunks(examples.len(), &examples, factory) + let correct = serialize_in_chunks(examples.len(), &examples, &idempotency_keys) .map(|res| res.unwrap().1) .flat_map(|body| serde_json::from_slice::(&body).unwrap().events) .collect::>(); for chunk_size in 1..examples.len() { - let actual = serialize_in_chunks(chunk_size, &examples, factory) + let actual = serialize_in_chunks(chunk_size, &examples, &idempotency_keys) .map(|res| res.unwrap().1) .flat_map(|body| serde_json::from_slice::(&body).unwrap().events) .collect::>(); From c43e664ff577d4568722e4e7a2b2c6267b609607 Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Tue, 3 Sep 2024 15:11:30 +0100 Subject: [PATCH 07/45] storcon: provide an az id in metadata.json from neon local (#8897) ## Problem Neon local set-up does not inject an az id in `metadata.json`. See real change in https://github.com/neondatabase/neon/pull/8852. ## Summary of changes We piggyback on the existing `availability_zone` pageserver configuration in order to avoid making neon local even more complex. --- control_plane/src/pageserver.rs | 23 ++++++++++++++++++- test_runner/fixtures/neon_fixtures.py | 12 +++++----- .../fixtures/pageserver/allowed_errors.py | 3 --- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/control_plane/src/pageserver.rs b/control_plane/src/pageserver.rs index 399b1c2653..31777eb7a5 100644 --- a/control_plane/src/pageserver.rs +++ b/control_plane/src/pageserver.rs @@ -181,6 +181,23 @@ impl PageServerNode { ); io::stdout().flush()?; + // If the config file we got as a CLI argument includes the `availability_zone` + // config, then use that to populate the `metadata.json` file for the pageserver. + // In production the deployment orchestrator does this for us. + let az_id = conf + .other + .get("availability_zone") + .map(|toml| { + let az_str = toml.to_string(); + // Trim the (") chars from the toml representation + if az_str.starts_with('"') && az_str.ends_with('"') { + az_str[1..az_str.len() - 1].to_string() + } else { + az_str + } + }) + .unwrap_or("local".to_string()); + let config = self .pageserver_init_make_toml(conf) .context("make pageserver toml")?; @@ -216,6 +233,7 @@ impl PageServerNode { let (_http_host, http_port) = parse_host_port(&self.conf.listen_http_addr).expect("Unable to parse listen_http_addr"); let http_port = http_port.unwrap_or(9898); + // Intentionally hand-craft JSON: this acts as an implicit format compat test // in case the pageserver-side structure is edited, and reflects the real life // situation: the metadata is written by some other script. @@ -226,7 +244,10 @@ impl PageServerNode { postgres_port: self.pg_connection_config.port(), http_host: "localhost".to_string(), http_port, - other: HashMap::new(), + other: HashMap::from([( + "availability_zone_id".to_string(), + serde_json::json!(az_id), + )]), }) .unwrap(), ) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 800ae03d13..0cbab71cc3 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1164,6 +1164,8 @@ class NeonEnv: "listen_http_addr": f"localhost:{pageserver_port.http}", "pg_auth_type": pg_auth_type, "http_auth_type": http_auth_type, + # Default which can be overriden with `NeonEnvBuilder.pageserver_config_override` + "availability_zone": "us-east-2a", } if self.pageserver_virtual_file_io_engine is not None: ps_cfg["virtual_file_io_engine"] = self.pageserver_virtual_file_io_engine @@ -1192,11 +1194,7 @@ class NeonEnv: # Create a corresponding NeonPageserver object self.pageservers.append( - NeonPageserver( - self, - ps_id, - port=pageserver_port, - ) + NeonPageserver(self, ps_id, port=pageserver_port, az_id=ps_cfg["availability_zone"]) ) cfg["pageservers"].append(ps_cfg) @@ -2400,6 +2398,7 @@ class NeonStorageController(MetricsGetter, LogUtils): "listen_http_port": node.service_port.http, "listen_pg_addr": "localhost", "listen_pg_port": node.service_port.pg, + "availability_zone_id": node.az_id, } log.info(f"node_register({body})") self.request( @@ -2923,10 +2922,11 @@ class NeonPageserver(PgProtocol, LogUtils): TEMP_FILE_SUFFIX = "___temp" - def __init__(self, env: NeonEnv, id: int, port: PageserverPort): + def __init__(self, env: NeonEnv, id: int, port: PageserverPort, az_id: str): super().__init__(host="localhost", port=port.pg, user="cloud_admin") self.env = env self.id = id + self.az_id = az_id self.running = False self.service_port = port self.version = env.get_binary_version("pageserver") diff --git a/test_runner/fixtures/pageserver/allowed_errors.py b/test_runner/fixtures/pageserver/allowed_errors.py index 70f2676245..f8d9a51c91 100755 --- a/test_runner/fixtures/pageserver/allowed_errors.py +++ b/test_runner/fixtures/pageserver/allowed_errors.py @@ -109,9 +109,6 @@ DEFAULT_STORAGE_CONTROLLER_ALLOWED_ERRORS = [ # controller's attempts to notify the endpoint). ".*reconciler.*neon_local notification hook failed.*", ".*reconciler.*neon_local error.*", - # Neon local does not provide pageserver with an AZ - # TODO: remove this once neon local does so - ".*registering without specific availability zone id.*", ] From 3916810f203cb086d4d6f6db760a39e5cffb0223 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Wed, 28 Aug 2024 17:39:13 +0300 Subject: [PATCH 08/45] safekeeper: add remote_path to Timeline It is used in many places, let's reduce number of ? on construction results. --- safekeeper/src/pull_timeline.rs | 6 +++--- safekeeper/src/timeline.rs | 8 +++++++- safekeeper/src/timeline_eviction.rs | 12 ++++-------- safekeeper/src/wal_backup.rs | 2 +- safekeeper/src/wal_backup_partial.rs | 10 ++-------- 5 files changed, 17 insertions(+), 21 deletions(-) diff --git a/safekeeper/src/pull_timeline.rs b/safekeeper/src/pull_timeline.rs index 1eacec9981..600a6bd8f0 100644 --- a/safekeeper/src/pull_timeline.rs +++ b/safekeeper/src/pull_timeline.rs @@ -183,10 +183,10 @@ impl WalResidentTimeline { "Replacing uploaded partial segment in in-mem control file: {replace:?}" ); - let remote_timeline_path = wal_backup::remote_timeline_path(&self.tli.ttid)?; + let remote_timeline_path = &self.tli.remote_path; wal_backup::copy_partial_segment( - &replace.previous.remote_path(&remote_timeline_path), - &replace.current.remote_path(&remote_timeline_path), + &replace.previous.remote_path(remote_timeline_path), + &replace.current.remote_path(remote_timeline_path), ) .await?; } diff --git a/safekeeper/src/timeline.rs b/safekeeper/src/timeline.rs index 57935d879f..f7c96d4f02 100644 --- a/safekeeper/src/timeline.rs +++ b/safekeeper/src/timeline.rs @@ -3,6 +3,7 @@ use anyhow::{anyhow, bail, Result}; use camino::Utf8PathBuf; +use remote_storage::RemotePath; use serde::{Deserialize, Serialize}; use tokio::fs::{self}; use tokio_util::sync::CancellationToken; @@ -36,7 +37,7 @@ use crate::state::{EvictionState, TimelineMemState, TimelinePersistentState, Tim use crate::timeline_guard::ResidenceGuard; use crate::timeline_manager::{AtomicStatus, ManagerCtl}; use crate::timelines_set::TimelinesSet; -use crate::wal_backup::{self}; +use crate::wal_backup::{self, remote_timeline_path}; use crate::wal_backup_partial::PartialRemoteSegment; use crate::{control_file, safekeeper::UNKNOWN_SERVER_VERSION}; @@ -469,6 +470,7 @@ impl From for ApiError { /// It also holds SharedState and provides mutually exclusive access to it. pub struct Timeline { pub ttid: TenantTimelineId, + pub remote_path: RemotePath, /// Used to broadcast commit_lsn updates to all background jobs. commit_lsn_watch_tx: watch::Sender, @@ -519,8 +521,10 @@ impl Timeline { let (shared_state_version_tx, shared_state_version_rx) = watch::channel(0); let walreceivers = WalReceivers::new(); + let remote_path = remote_timeline_path(&ttid)?; Ok(Timeline { ttid, + remote_path, commit_lsn_watch_tx, commit_lsn_watch_rx, term_flush_lsn_watch_tx, @@ -557,8 +561,10 @@ impl Timeline { TimelinePersistentState::new(&ttid, server_info, vec![], commit_lsn, local_start_lsn); let walreceivers = WalReceivers::new(); + let remote_path = remote_timeline_path(&ttid)?; Ok(Timeline { ttid, + remote_path, commit_lsn_watch_tx, commit_lsn_watch_rx, term_flush_lsn_watch_tx, diff --git a/safekeeper/src/timeline_eviction.rs b/safekeeper/src/timeline_eviction.rs index ae6f3f4b7e..2ccb058720 100644 --- a/safekeeper/src/timeline_eviction.rs +++ b/safekeeper/src/timeline_eviction.rs @@ -167,7 +167,7 @@ async fn redownload_partial_segment( partial: &PartialRemoteSegment, ) -> anyhow::Result<()> { let tmp_file = mgr.tli.timeline_dir().join("remote_partial.tmp"); - let remote_segfile = remote_segment_path(mgr, partial)?; + let remote_segfile = remote_segment_path(mgr, partial); debug!( "redownloading partial segment: {} -> {}", @@ -252,7 +252,7 @@ async fn do_validation( ); } - let remote_segfile = remote_segment_path(mgr, partial)?; + let remote_segfile = remote_segment_path(mgr, partial); let mut remote_reader: std::pin::Pin> = wal_backup::read_object(&remote_segfile, 0).await?; @@ -279,12 +279,8 @@ fn local_segment_path(mgr: &Manager, partial: &PartialRemoteSegment) -> Utf8Path local_partial_segfile } -fn remote_segment_path( - mgr: &Manager, - partial: &PartialRemoteSegment, -) -> anyhow::Result { - let remote_timeline_path = wal_backup::remote_timeline_path(&mgr.tli.ttid)?; - Ok(partial.remote_path(&remote_timeline_path)) +fn remote_segment_path(mgr: &Manager, partial: &PartialRemoteSegment) -> RemotePath { + partial.remote_path(&mgr.tli.remote_path) } /// Compare first `n` bytes of two readers. If the bytes differ, return an error. diff --git a/safekeeper/src/wal_backup.rs b/safekeeper/src/wal_backup.rs index aa1a6696a1..1c9ec5c007 100644 --- a/safekeeper/src/wal_backup.rs +++ b/safekeeper/src/wal_backup.rs @@ -315,7 +315,7 @@ async fn backup_lsn_range( anyhow::bail!("parallel_jobs must be >= 1"); } - let remote_timeline_path = remote_timeline_path(&timeline.ttid)?; + let remote_timeline_path = &timeline.remote_path; let start_lsn = *backup_lsn; let segments = get_segments(start_lsn, end_lsn, wal_seg_size); diff --git a/safekeeper/src/wal_backup_partial.rs b/safekeeper/src/wal_backup_partial.rs index 675a051887..4022c9409b 100644 --- a/safekeeper/src/wal_backup_partial.rs +++ b/safekeeper/src/wal_backup_partial.rs @@ -31,7 +31,7 @@ use crate::{ safekeeper::Term, timeline::WalResidentTimeline, timeline_manager::StateSnapshot, - wal_backup::{self, remote_timeline_path}, + wal_backup::{self}, SafeKeeperConf, }; @@ -388,13 +388,7 @@ pub async fn main_task( let wal_seg_size = tli.get_wal_seg_size().await; let local_prefix = tli.get_timeline_dir(); - let remote_timeline_path = match remote_timeline_path(&tli.ttid) { - Ok(path) => path, - Err(e) => { - error!("failed to create remote path: {:?}", e); - return None; - } - }; + let remote_timeline_path = tli.remote_path.clone(); let mut backup = PartialBackup { wal_seg_size, From 80512e2779f40af7602fe3221ccc7eaa0499e61e Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Fri, 30 Aug 2024 12:35:41 +0300 Subject: [PATCH 09/45] safekeeper: add endpoint resetting uploaded partial segment state. Endpoint implementation sends msg to manager requesting to do the reset. Manager stops current partial backup upload task if it exists and performs the reset. Also slightly tweak eviction condition: all full segments before flush_lsn must be uploaded (and committed) and there must be only one segment left on disk (partial). This allows to evict timelines which started not on the first segment and didn't fill the whole segment (previous condition wasn't good because last_removed_segno was 0). ref https://github.com/neondatabase/neon/issues/8759 --- safekeeper/src/http/routes.rs | 23 ++++++ safekeeper/src/timeline.rs | 4 + safekeeper/src/timeline_eviction.rs | 21 +++-- safekeeper/src/timeline_manager.rs | 88 ++++++++++++++++++-- safekeeper/src/wal_backup.rs | 6 +- safekeeper/src/wal_backup_partial.rs | 101 +++++++++++++++++------ test_runner/fixtures/neon_fixtures.py | 6 +- test_runner/fixtures/safekeeper/http.py | 24 ++++++ test_runner/regress/test_wal_acceptor.py | 99 +++++++++++++++++++++- 9 files changed, 325 insertions(+), 47 deletions(-) diff --git a/safekeeper/src/http/routes.rs b/safekeeper/src/http/routes.rs index 91ffa95c21..9b7424a818 100644 --- a/safekeeper/src/http/routes.rs +++ b/safekeeper/src/http/routes.rs @@ -389,6 +389,25 @@ async fn timeline_digest_handler(request: Request) -> Result) -> Result, ApiError> { + let ttid = TenantTimelineId::new( + parse_request_param(&request, "tenant_id")?, + parse_request_param(&request, "timeline_id")?, + ); + check_permission(&request, Some(ttid.tenant_id))?; + + let tli = GlobalTimelines::get(ttid).map_err(ApiError::from)?; + + let response = tli + .backup_partial_reset() + .await + .map_err(ApiError::InternalServerError)?; + json_response(StatusCode::OK, response) +} + /// Used only in tests to hand craft required data. async fn record_safekeeper_info(mut request: Request) -> Result, ApiError> { let ttid = TenantTimelineId::new( @@ -607,6 +626,10 @@ pub fn make_router(conf: SafeKeeperConf) -> RouterBuilder .get("/v1/tenant/:tenant_id/timeline/:timeline_id/digest", |r| { request_span(r, timeline_digest_handler) }) + .post( + "/v1/tenant/:tenant_id/timeline/:timeline_id/backup_partial_reset", + |r| request_span(r, timeline_backup_partial_reset), + ) .post("/v1/record_safekeeper_info/:tenant_id/:timeline_id", |r| { request_span(r, record_safekeeper_info) }) diff --git a/safekeeper/src/timeline.rs b/safekeeper/src/timeline.rs index f7c96d4f02..95ee925e1a 100644 --- a/safekeeper/src/timeline.rs +++ b/safekeeper/src/timeline.rs @@ -908,6 +908,10 @@ impl Timeline { Ok(WalResidentTimeline::new(self.clone(), guard)) } + + pub async fn backup_partial_reset(self: &Arc) -> Result> { + self.manager_ctl.backup_partial_reset().await + } } /// This is a guard that allows to read/write disk timeline state. diff --git a/safekeeper/src/timeline_eviction.rs b/safekeeper/src/timeline_eviction.rs index 2ccb058720..5d0567575c 100644 --- a/safekeeper/src/timeline_eviction.rs +++ b/safekeeper/src/timeline_eviction.rs @@ -28,28 +28,38 @@ impl Manager { /// - control file is flushed (no next event scheduled) /// - no WAL residence guards /// - no pushes to the broker - /// - partial WAL backup is uploaded + /// - last partial WAL segment is uploaded + /// - all local segments before the uploaded partial are committed and uploaded pub(crate) fn ready_for_eviction( &self, next_event: &Option, state: &StateSnapshot, ) -> bool { - self.backup_task.is_none() + let ready = self.backup_task.is_none() && self.recovery_task.is_none() && self.wal_removal_task.is_none() && self.partial_backup_task.is_none() - && self.partial_backup_uploaded.is_some() && next_event.is_none() && self.access_service.is_empty() && !self.tli_broker_active.get() + // Partial segment of current flush_lsn is uploaded up to this flush_lsn. && !wal_backup_partial::needs_uploading(state, &self.partial_backup_uploaded) + // And it is the next one after the last removed. Given that local + // WAL is removed only after it is uploaded to s3 (and pageserver + // advancing remote_consistent_lsn) which happens only after WAL is + // committed, true means all this is done. + // + // This also works for the first segment despite last_removed_segno + // being 0 on init because this 0 triggers run of wal_removal_task + // on success of which manager updates the horizon. && self .partial_backup_uploaded .as_ref() .unwrap() .flush_lsn .segment_number(self.wal_seg_size) - == self.last_removed_segno + 1 + == self.last_removed_segno + 1; + ready } /// Evict the timeline to remote storage. @@ -83,7 +93,8 @@ impl Manager { info!("successfully evicted timeline"); } - /// Restore evicted timeline from remote storage. + /// Attempt to restore evicted timeline from remote storage; it must be + /// offloaded. #[instrument(name = "unevict_timeline", skip_all)] pub(crate) async fn unevict_timeline(&mut self) { assert!(self.is_offloaded); diff --git a/safekeeper/src/timeline_manager.rs b/safekeeper/src/timeline_manager.rs index 482614fac7..f997f48454 100644 --- a/safekeeper/src/timeline_manager.rs +++ b/safekeeper/src/timeline_manager.rs @@ -11,12 +11,14 @@ use std::{ time::Duration, }; +use futures::channel::oneshot; use postgres_ffi::XLogSegNo; use serde::{Deserialize, Serialize}; use tokio::{ task::{JoinError, JoinHandle}, time::Instant, }; +use tokio_util::sync::CancellationToken; use tracing::{debug, info, info_span, instrument, warn, Instrument}; use utils::lsn::Lsn; @@ -33,7 +35,7 @@ use crate::{ timeline_guard::{AccessService, GuardId, ResidenceGuard}, timelines_set::{TimelineSetGuard, TimelinesSet}, wal_backup::{self, WalBackupTaskHandle}, - wal_backup_partial::{self, PartialRemoteSegment}, + wal_backup_partial::{self, PartialBackup, PartialRemoteSegment}, SafeKeeperConf, }; @@ -96,6 +98,8 @@ pub enum ManagerCtlMessage { GuardRequest(tokio::sync::oneshot::Sender>), /// Request to drop the guard. GuardDrop(GuardId), + /// Request to reset uploaded partial backup state. + BackupPartialReset(oneshot::Sender>>), } impl std::fmt::Debug for ManagerCtlMessage { @@ -103,6 +107,7 @@ impl std::fmt::Debug for ManagerCtlMessage { match self { ManagerCtlMessage::GuardRequest(_) => write!(f, "GuardRequest"), ManagerCtlMessage::GuardDrop(id) => write!(f, "GuardDrop({:?})", id), + ManagerCtlMessage::BackupPartialReset(_) => write!(f, "BackupPartialReset"), } } } @@ -143,6 +148,19 @@ impl ManagerCtl { .and_then(std::convert::identity) } + /// Request timeline manager to reset uploaded partial segment state and + /// wait for the result. + pub async fn backup_partial_reset(&self) -> anyhow::Result> { + let (tx, rx) = oneshot::channel(); + self.manager_tx + .send(ManagerCtlMessage::BackupPartialReset(tx)) + .expect("manager task is not running"); + match rx.await { + Ok(res) => res, + Err(_) => anyhow::bail!("timeline manager is gone"), + } + } + /// Must be called exactly once to bootstrap the manager. pub fn bootstrap_manager( &self, @@ -181,7 +199,8 @@ pub(crate) struct Manager { pub(crate) wal_removal_task: Option>>, // partial backup - pub(crate) partial_backup_task: Option>>, + pub(crate) partial_backup_task: + Option<(JoinHandle>, CancellationToken)>, pub(crate) partial_backup_uploaded: Option, // misc @@ -302,12 +321,12 @@ pub async fn main_task( _ = sleep_until(&next_event) => { // we were waiting for some event (e.g. cfile save) } - res = await_task_finish(&mut mgr.wal_removal_task) => { + res = await_task_finish(mgr.wal_removal_task.as_mut()) => { // WAL removal task finished mgr.wal_removal_task = None; mgr.update_wal_removal_end(res); } - res = await_task_finish(&mut mgr.partial_backup_task) => { + res = await_task_finish(mgr.partial_backup_task.as_mut().map(|(handle, _)| handle)) => { // partial backup task finished mgr.partial_backup_task = None; mgr.update_partial_backup_end(res); @@ -335,8 +354,9 @@ pub async fn main_task( } } - if let Some(partial_backup_task) = &mut mgr.partial_backup_task { - if let Err(e) = partial_backup_task.await { + if let Some((handle, cancel)) = &mut mgr.partial_backup_task { + cancel.cancel(); + if let Err(e) = handle.await { warn!("partial backup task failed: {:?}", e); } } @@ -560,11 +580,14 @@ impl Manager { } // Get WalResidentTimeline and start partial backup task. - self.partial_backup_task = Some(tokio::spawn(wal_backup_partial::main_task( + let cancel = CancellationToken::new(); + let handle = tokio::spawn(wal_backup_partial::main_task( self.wal_resident_timeline(), self.conf.clone(), self.global_rate_limiter.clone(), - ))); + cancel.clone(), + )); + self.partial_backup_task = Some((handle, cancel)); } /// Update the state after partial WAL backup task finished. @@ -579,6 +602,39 @@ impl Manager { } } + /// Reset partial backup state and remove its remote storage data. Since it + /// might concurrently uploading something, cancel the task first. + async fn backup_partial_reset(&mut self) -> anyhow::Result> { + info!("resetting partial backup state"); + // Force unevict timeline if it is evicted before erasing partial backup + // state. The intended use of this function is to drop corrupted remote + // state; we haven't enabled local files deletion yet anywhere, + // so direct switch is safe. + if self.is_offloaded { + self.tli.switch_to_present().await?; + // switch manager state as soon as possible + self.is_offloaded = false; + } + + if let Some((handle, cancel)) = &mut self.partial_backup_task { + cancel.cancel(); + info!("cancelled partial backup task, awaiting it"); + // we're going to reset .partial_backup_uploaded to None anyway, so ignore the result + handle.await.ok(); + self.partial_backup_task = None; + } + + let tli = self.wal_resident_timeline(); + let mut partial_backup = PartialBackup::new(tli, self.conf.clone()).await; + // Reset might fail e.g. when cfile is already reset but s3 removal + // failed, so set manager state to None beforehand. In any case caller + // is expected to retry until success. + self.partial_backup_uploaded = None; + let res = partial_backup.reset().await?; + info!("reset is done"); + Ok(res) + } + /// Handle message arrived from ManagerCtl. async fn handle_message(&mut self, msg: Option) { debug!("received manager message: {:?}", msg); @@ -602,6 +658,16 @@ impl Manager { Some(ManagerCtlMessage::GuardDrop(guard_id)) => { self.access_service.drop_guard(guard_id); } + Some(ManagerCtlMessage::BackupPartialReset(tx)) => { + info!("resetting uploaded partial backup state"); + let res = self.backup_partial_reset().await; + if let Err(ref e) = res { + warn!("failed to reset partial backup state: {:?}", e); + } + if tx.send(res).is_err() { + warn!("failed to send partial backup reset result, receiver dropped"); + } + } None => { // can't happen, we're holding the sender unreachable!(); @@ -619,7 +685,11 @@ async fn sleep_until(option: &Option) { } } -async fn await_task_finish(option: &mut Option>) -> Result { +/// Future that resolves when the task is finished or never if the task is None. +/// +/// Note: it accepts Option<&mut> instead of &mut Option<> because mapping the +/// option to get the latter is hard. +async fn await_task_finish(option: Option<&mut JoinHandle>) -> Result { if let Some(task) = option { task.await } else { diff --git a/safekeeper/src/wal_backup.rs b/safekeeper/src/wal_backup.rs index 1c9ec5c007..95012bb004 100644 --- a/safekeeper/src/wal_backup.rs +++ b/safekeeper/src/wal_backup.rs @@ -328,11 +328,7 @@ async fn backup_lsn_range( loop { let added_task = match iter.next() { Some(s) => { - uploads.push_back(backup_single_segment( - s, - timeline_dir, - &remote_timeline_path, - )); + uploads.push_back(backup_single_segment(s, timeline_dir, remote_timeline_path)); true } None => false, diff --git a/safekeeper/src/wal_backup_partial.rs b/safekeeper/src/wal_backup_partial.rs index 4022c9409b..4f320f43f8 100644 --- a/safekeeper/src/wal_backup_partial.rs +++ b/safekeeper/src/wal_backup_partial.rs @@ -22,6 +22,7 @@ use postgres_ffi::{XLogFileName, XLogSegNo, PG_TLI}; use remote_storage::RemotePath; use serde::{Deserialize, Serialize}; +use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, instrument, warn}; use utils::{id::NodeId, lsn::Lsn}; @@ -145,7 +146,7 @@ impl State { } } -struct PartialBackup { +pub struct PartialBackup { wal_seg_size: usize, tli: WalResidentTimeline, conf: SafeKeeperConf, @@ -155,8 +156,25 @@ struct PartialBackup { state: State, } -// Read-only methods for getting segment names impl PartialBackup { + pub async fn new(tli: WalResidentTimeline, conf: SafeKeeperConf) -> PartialBackup { + let (_, persistent_state) = tli.get_state().await; + let wal_seg_size = tli.get_wal_seg_size().await; + + let local_prefix = tli.get_timeline_dir(); + let remote_timeline_path = tli.remote_path.clone(); + + PartialBackup { + wal_seg_size, + tli, + state: persistent_state.partial_backup, + conf, + local_prefix, + remote_timeline_path, + } + } + + // Read-only methods for getting segment names fn segno(&self, lsn: Lsn) -> XLogSegNo { lsn.segment_number(self.wal_seg_size) } @@ -297,6 +315,18 @@ impl PartialBackup { Ok(()) } + // Prepend to the given segments remote prefix and delete them from the + // remote storage. + async fn delete_segments(&self, segments_to_delete: &Vec) -> anyhow::Result<()> { + info!("deleting objects: {:?}", segments_to_delete); + let mut objects_to_delete = vec![]; + for seg in segments_to_delete.iter() { + let remote_path = self.remote_timeline_path.join(seg); + objects_to_delete.push(remote_path); + } + wal_backup::delete_objects(&objects_to_delete).await + } + /// Delete all non-Uploaded segments from the remote storage. There should be only one /// Uploaded segment at a time. #[instrument(name = "gc", skip_all)] @@ -329,15 +359,8 @@ impl PartialBackup { ); } - info!("deleting objects: {:?}", segments_to_delete); - let mut objects_to_delete = vec![]; - for seg in segments_to_delete.iter() { - let remote_path = self.remote_timeline_path.join(seg); - objects_to_delete.push(remote_path); - } - - // removing segments from remote storage - wal_backup::delete_objects(&objects_to_delete).await?; + // execute the deletion + self.delete_segments(&segments_to_delete).await?; // now we can update the state on disk let new_state = { @@ -349,6 +372,27 @@ impl PartialBackup { Ok(()) } + + /// Remove uploaded segment(s) from the state and remote storage. Aimed for + /// manual intervention, not normally needed. + /// Returns list of segments which potentially existed in the remote storage. + pub async fn reset(&mut self) -> anyhow::Result> { + let segments_to_delete = self + .state + .segments + .iter() + .map(|seg| seg.name.clone()) + .collect(); + + // First reset cfile state, and only then objects themselves. If the + // later fails we might leave some garbage behind; that's ok for this + // single time usage. + let new_state = State { segments: vec![] }; + self.commit_state(new_state).await?; + + self.delete_segments(&segments_to_delete).await?; + Ok(segments_to_delete) + } } /// Check if everything is uploaded and partial backup task doesn't need to run. @@ -377,27 +421,16 @@ pub async fn main_task( tli: WalResidentTimeline, conf: SafeKeeperConf, limiter: RateLimiter, + cancel: CancellationToken, ) -> Option { debug!("started"); let await_duration = conf.partial_backup_timeout; let mut first_iteration = true; - let (_, persistent_state) = tli.get_state().await; let mut commit_lsn_rx = tli.get_commit_lsn_watch_rx(); let mut flush_lsn_rx = tli.get_term_flush_lsn_watch_rx(); - let wal_seg_size = tli.get_wal_seg_size().await; - let local_prefix = tli.get_timeline_dir(); - let remote_timeline_path = tli.remote_path.clone(); - - let mut backup = PartialBackup { - wal_seg_size, - tli, - state: persistent_state.partial_backup, - conf, - local_prefix, - remote_timeline_path, - }; + let mut backup = PartialBackup::new(tli, conf).await; debug!("state: {:?}", backup.state); @@ -427,6 +460,10 @@ pub async fn main_task( && flush_lsn_rx.borrow().term == seg.term { // we have nothing to do, the last segment is already uploaded + debug!( + "exiting, uploaded up to term={} flush_lsn={} commit_lsn={}", + seg.term, seg.flush_lsn, seg.commit_lsn + ); return Some(seg.clone()); } } @@ -438,6 +475,10 @@ pub async fn main_task( info!("timeline canceled"); return None; } + _ = cancel.cancelled() => { + info!("task canceled"); + return None; + } _ = flush_lsn_rx.changed() => {} } } @@ -464,6 +505,10 @@ pub async fn main_task( info!("timeline canceled"); return None; } + _ = cancel.cancelled() => { + info!("task canceled"); + return None; + } _ = commit_lsn_rx.changed() => {} _ = flush_lsn_rx.changed() => { let segno = backup.segno(flush_lsn_rx.borrow().lsn); @@ -486,7 +531,13 @@ pub async fn main_task( } // limit concurrent uploads - let _upload_permit = limiter.acquire_partial_backup().await; + let _upload_permit = tokio::select! { + acq = limiter.acquire_partial_backup() => acq, + _ = cancel.cancelled() => { + info!("task canceled"); + return None; + } + }; let prepared = backup.prepare_upload().await; if let Some(seg) = &uploaded_segment { diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 0cbab71cc3..8c99408cfb 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -4553,6 +4553,8 @@ class Safekeeper(LogUtils): def timeline_dir(self, tenant_id, timeline_id) -> Path: return self.data_dir / str(tenant_id) / str(timeline_id) + # List partial uploaded segments of this safekeeper. Works only for + # RemoteStorageKind.LOCAL_FS. def list_uploaded_segments(self, tenant_id: TenantId, timeline_id: TimelineId): tline_path = ( self.env.repo_dir @@ -4562,9 +4564,11 @@ class Safekeeper(LogUtils): / str(timeline_id) ) assert isinstance(self.env.safekeepers_remote_storage, LocalFsStorage) - return self._list_segments_in_dir( + segs = self._list_segments_in_dir( tline_path, lambda name: ".metadata" not in name and ".___temp" not in name ) + mysegs = [s for s in segs if f"sk{self.id}" in s] + return mysegs def list_segments(self, tenant_id, timeline_id) -> List[str]: """ diff --git a/test_runner/fixtures/safekeeper/http.py b/test_runner/fixtures/safekeeper/http.py index 05b43cfb72..9bf03554e7 100644 --- a/test_runner/fixtures/safekeeper/http.py +++ b/test_runner/fixtures/safekeeper/http.py @@ -174,6 +174,22 @@ class SafekeeperHttpClient(requests.Session, MetricsGetter): assert isinstance(res_json, dict) return res_json + def debug_dump_timeline( + self, timeline_id: TimelineId, params: Optional[Dict[str, str]] = None + ) -> Any: + params = params or {} + params["timeline_id"] = str(timeline_id) + dump = self.debug_dump(params) + return dump["timelines"][0] + + def get_partial_backup(self, timeline_id: TimelineId) -> Any: + dump = self.debug_dump_timeline(timeline_id, {"dump_control_file": "true"}) + return dump["control_file"]["partial_backup"] + + def get_eviction_state(self, timeline_id: TimelineId) -> Any: + dump = self.debug_dump_timeline(timeline_id, {"dump_control_file": "true"}) + return dump["control_file"]["eviction_state"] + def pull_timeline(self, body: Dict[str, Any]) -> Dict[str, Any]: res = self.post(f"http://localhost:{self.port}/v1/pull_timeline", json=body) res.raise_for_status() @@ -228,6 +244,14 @@ class SafekeeperHttpClient(requests.Session, MetricsGetter): assert isinstance(res_json, dict) return res_json + def backup_partial_reset(self, tenant_id: TenantId, timeline_id: TimelineId): + res = self.post( + f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/backup_partial_reset", + json={}, + ) + res.raise_for_status() + return res.json() + def record_safekeeper_info(self, tenant_id: TenantId, timeline_id: TimelineId, body): res = self.post( f"http://localhost:{self.port}/v1/record_safekeeper_info/{tenant_id}/{timeline_id}", diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index 19df834b81..3785651aed 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -72,6 +72,17 @@ def wait_lsn_force_checkpoint( wait_lsn_force_checkpoint_at(lsn, tenant_id, timeline_id, ps, pageserver_conn_options) +def wait_lsn_force_checkpoint_at_sk( + safekeeper: Safekeeper, + tenant_id: TenantId, + timeline_id: TimelineId, + ps: NeonPageserver, + pageserver_conn_options=None, +): + sk_flush_lsn = safekeeper.get_flush_lsn(tenant_id, timeline_id) + wait_lsn_force_checkpoint_at(sk_flush_lsn, tenant_id, timeline_id, ps, pageserver_conn_options) + + def wait_lsn_force_checkpoint_at( lsn: Lsn, tenant_id: TenantId, @@ -79,6 +90,10 @@ def wait_lsn_force_checkpoint_at( ps: NeonPageserver, pageserver_conn_options=None, ): + """ + Wait until pageserver receives given lsn, force checkpoint and wait for + upload, i.e. remote_consistent_lsn advancement. + """ pageserver_conn_options = pageserver_conn_options or {} auth_token = None @@ -2330,6 +2345,77 @@ def test_s3_eviction( assert event_metrics_seen +# Test resetting uploaded partial segment state. +def test_backup_partial_reset(neon_env_builder: NeonEnvBuilder): + neon_env_builder.num_safekeepers = 1 + neon_env_builder.enable_safekeeper_remote_storage(default_remote_storage()) + # We want to upload/evict quickly, but not too quickly to check that s3 is + # empty before next round of upload happens. + # Note: this test fails with --delete-offloaded-wal, this is expected. + neon_env_builder.safekeeper_extra_opts = [ + "--enable-offload", + "--partial-backup-timeout", + "1s", + "--control-file-save-interval", + "1s", + "--eviction-min-resident=1s", + ] + # XXX: pageserver currently connects to safekeeper as long as connection + # manager doesn't remove its entry (default lagging_wal_timeout is 10s), + # causing uneviction. It should be fixed to not reconnect if last + # remote_consistent_lsn is communicated and there is nothing to fetch. Make + # value lower to speed up the test. + initial_tenant_conf = { + "lagging_wal_timeout": "1s", + } + env = neon_env_builder.init_start(initial_tenant_conf) + + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline + + endpoint = env.endpoints.create("main") + endpoint.start() + endpoint.safe_psql("create table t(key int, value text)") + endpoint.stop() + sk = env.safekeepers[0] + # eviction won't happen until remote_consistent_lsn catches up. + wait_lsn_force_checkpoint_at_sk(sk, tenant_id, timeline_id, env.pageserver) + + http_cli = env.safekeepers[0].http_client() + + # wait until eviction happens + def evicted(): + eviction_state = http_cli.get_eviction_state(timeline_id) + log.info(f"eviction_state: {eviction_state}") + if isinstance(eviction_state, str) and eviction_state == "Present": + raise Exception("eviction didn't happen yet") + + wait_until(30, 1, evicted) + # it must have uploaded something + uploaded_segs = sk.list_uploaded_segments(tenant_id, timeline_id) + log.info(f"uploaded segments before reset: {uploaded_segs}") + assert len(uploaded_segs) > 0 + + reset_res = http_cli.backup_partial_reset(tenant_id, timeline_id) + log.info(f"reset res: {reset_res}") + + # Backup_partial_reset must have reset the state and dropped s3 segment. + # + # Note: if listing takes more than --partial-backup-timeout test becomes + # flaky because file might be reuploaded. With local fs it shouldn't be an + # issue, but can add retry if this appears. + uploaded_segs = sk.list_uploaded_segments(tenant_id, timeline_id) + log.info(f"uploaded segments after reset: {uploaded_segs}") + assert len(uploaded_segs) == 0 + + # calling second time should be ok + http_cli.backup_partial_reset(tenant_id, timeline_id) + + # inserting data should be ok + endpoint.start() + endpoint.safe_psql("insert into t values(1, 'hehe')") + + def test_pull_timeline_partial_segment_integrity(neon_env_builder: NeonEnvBuilder): """ Verify that pulling timeline from a SK with an uploaded partial segment @@ -2357,7 +2443,16 @@ def test_pull_timeline_partial_segment_integrity(neon_env_builder: NeonEnvBuilde "--eviction-min-resident=500ms", ] - env = neon_env_builder.init_start(initial_tenant_conf={"checkpoint_timeout": "100ms"}) + # XXX: pageserver currently connects to safekeeper as long as connection + # manager doesn't remove its entry (default lagging_wal_timeout is 10s), + # causing uneviction. It should be fixed to not reconnect if last + # remote_consistent_lsn is communicated and there is nothing to fetch. Until + # this is fixed make value lower to speed up the test. + initial_tenant_conf = { + "lagging_wal_timeout": "1s", + "checkpoint_timeout": "100ms", + } + env = neon_env_builder.init_start(initial_tenant_conf=initial_tenant_conf) tenant_id = env.initial_tenant timeline_id = env.initial_timeline @@ -2421,7 +2516,7 @@ def test_pull_timeline_partial_segment_integrity(neon_env_builder: NeonEnvBuilde endpoint.start(safekeepers=[2, 3]) def new_partial_segment_uploaded(): - segs = src_sk.list_uploaded_segments(tenant_id, timeline_id) + segs = dst_sk.list_uploaded_segments(tenant_id, timeline_id) for seg in segs: if "partial" in seg and "sk3" in seg: return seg From 83dd7f559c16aa0ed546b9fa6d78e04d32a01de1 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Tue, 3 Sep 2024 15:35:59 +0300 Subject: [PATCH 10/45] safekeeper: more consistent task naming. Make all them snake case. --- safekeeper/src/broker.rs | 2 +- safekeeper/src/recovery.rs | 2 +- safekeeper/src/wal_backup.rs | 2 +- safekeeper/src/wal_backup_partial.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/safekeeper/src/broker.rs b/safekeeper/src/broker.rs index 7cc2142291..485816408f 100644 --- a/safekeeper/src/broker.rs +++ b/safekeeper/src/broker.rs @@ -86,7 +86,7 @@ async fn push_loop(conf: SafeKeeperConf) -> anyhow::Result<()> { } /// Subscribe and fetch all the interesting data from the broker. -#[instrument(name = "broker pull", skip_all)] +#[instrument(name = "broker_pull", skip_all)] async fn pull_loop(conf: SafeKeeperConf, stats: Arc) -> Result<()> { let mut client = storage_broker::connect(conf.broker_endpoint, conf.broker_keepalive_interval)?; diff --git a/safekeeper/src/recovery.rs b/safekeeper/src/recovery.rs index a59ff07b96..9c4149d8f1 100644 --- a/safekeeper/src/recovery.rs +++ b/safekeeper/src/recovery.rs @@ -35,7 +35,7 @@ use crate::{ /// Entrypoint for per timeline task which always runs, checking whether /// recovery for this safekeeper is needed and starting it if so. -#[instrument(name = "recovery task", skip_all, fields(ttid = %tli.ttid))] +#[instrument(name = "recovery", skip_all, fields(ttid = %tli.ttid))] pub async fn recovery_main(tli: WalResidentTimeline, conf: SafeKeeperConf) { info!("started"); diff --git a/safekeeper/src/wal_backup.rs b/safekeeper/src/wal_backup.rs index 95012bb004..ef26ac99c5 100644 --- a/safekeeper/src/wal_backup.rs +++ b/safekeeper/src/wal_backup.rs @@ -203,7 +203,7 @@ struct WalBackupTask { } /// Offload single timeline. -#[instrument(name = "WAL backup", skip_all, fields(ttid = %tli.ttid))] +#[instrument(name = "wal_backup", skip_all, fields(ttid = %tli.ttid))] async fn backup_task_main( tli: WalResidentTimeline, parallel_jobs: usize, diff --git a/safekeeper/src/wal_backup_partial.rs b/safekeeper/src/wal_backup_partial.rs index 4f320f43f8..4050a82fff 100644 --- a/safekeeper/src/wal_backup_partial.rs +++ b/safekeeper/src/wal_backup_partial.rs @@ -416,7 +416,7 @@ pub(crate) fn needs_uploading( /// /// When there is nothing more to do and the last segment was successfully uploaded, the task /// returns PartialRemoteSegment, to signal readiness for offloading the timeline. -#[instrument(name = "Partial backup", skip_all, fields(ttid = %tli.ttid))] +#[instrument(name = "partial_backup", skip_all, fields(ttid = %tli.ttid))] pub async fn main_task( tli: WalResidentTimeline, conf: SafeKeeperConf, From c7187be8a11a43a0bc74d8745912df4a6c5c1db7 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Wed, 7 Aug 2024 19:26:06 +0300 Subject: [PATCH 11/45] safekeeper: check for non-consecutive writes in safekeeper.rs wal_storage.rs already checks this, but since this is a quite legit scenario check it at safekeeper.rs (consensus level) as well. ref https://github.com/neondatabase/neon/issues/8212 This is a take 2; previous PR #8640 had been reverted because interplay with another change broke test_last_log_term_switch. --- safekeeper/src/safekeeper.rs | 126 ++++++++++++++---- safekeeper/src/wal_storage.rs | 6 + .../tests/walproposer_sim/safekeeper_disk.rs | 4 + 3 files changed, 113 insertions(+), 23 deletions(-) diff --git a/safekeeper/src/safekeeper.rs b/safekeeper/src/safekeeper.rs index 486954c7b9..dbe0034de2 100644 --- a/safekeeper/src/safekeeper.rs +++ b/safekeeper/src/safekeeper.rs @@ -875,6 +875,29 @@ where return Ok(Some(AcceptorProposerMessage::AppendResponse(resp))); } + // Disallow any non-sequential writes, which can result in gaps or + // overwrites. If we need to move the pointer, ProposerElected message + // should have truncated WAL first accordingly. Note that the first + // condition (WAL rewrite) is quite expected in real world; it happens + // when walproposer reconnects to safekeeper and writes some more data + // while first connection still gets some packets later. It might be + // better to not log this as error! above. + let write_lsn = self.wal_store.write_lsn(); + if write_lsn > msg.h.begin_lsn { + bail!( + "append request rewrites WAL written before, write_lsn={}, msg lsn={}", + write_lsn, + msg.h.begin_lsn + ); + } + if write_lsn < msg.h.begin_lsn && write_lsn != Lsn(0) { + bail!( + "append request creates gap in written WAL, write_lsn={}, msg lsn={}", + write_lsn, + msg.h.begin_lsn, + ); + } + // Now we know that we are in the same term as the proposer, // processing the message. @@ -960,10 +983,7 @@ mod tests { use postgres_ffi::{XLogSegNo, WAL_SEGMENT_SIZE}; use super::*; - use crate::{ - state::{EvictionState, PersistedPeers, TimelinePersistentState}, - wal_storage::Storage, - }; + use crate::state::{EvictionState, PersistedPeers, TimelinePersistentState}; use std::{ops::Deref, str::FromStr, time::Instant}; // fake storage for tests @@ -1003,6 +1023,10 @@ mod tests { } impl wal_storage::Storage for DummyWalStore { + fn write_lsn(&self) -> Lsn { + self.lsn + } + fn flush_lsn(&self) -> Lsn { self.lsn } @@ -1076,7 +1100,7 @@ mod tests { let mut sk = SafeKeeper::new(TimelineState::new(storage), wal_store, NodeId(0)).unwrap(); let mut ar_hdr = AppendRequestHeader { - term: 1, + term: 2, term_start_lsn: Lsn(3), begin_lsn: Lsn(1), end_lsn: Lsn(2), @@ -1090,24 +1114,29 @@ mod tests { }; let pem = ProposerElected { - term: 1, - start_streaming_at: Lsn(3), - term_history: TermHistory(vec![TermLsn { - term: 1, - lsn: Lsn(3), - }]), - timeline_start_lsn: Lsn(0), + term: 2, + start_streaming_at: Lsn(1), + term_history: TermHistory(vec![ + TermLsn { + term: 1, + lsn: Lsn(1), + }, + TermLsn { + term: 2, + lsn: Lsn(3), + }, + ]), + timeline_start_lsn: Lsn(1), }; sk.process_msg(&ProposerAcceptorMessage::Elected(pem)) .await .unwrap(); // check that AppendRequest before term_start_lsn doesn't switch last_log_term. - let resp = sk - .process_msg(&ProposerAcceptorMessage::AppendRequest(append_request)) - .await; - assert!(resp.is_ok()); - assert_eq!(sk.get_last_log_term(), 0); + sk.process_msg(&ProposerAcceptorMessage::AppendRequest(append_request)) + .await + .unwrap(); + assert_eq!(sk.get_last_log_term(), 1); // but record at term_start_lsn does the switch ar_hdr.begin_lsn = Lsn(2); @@ -1116,12 +1145,63 @@ mod tests { h: ar_hdr, wal_data: Bytes::from_static(b"b"), }; - let resp = sk - .process_msg(&ProposerAcceptorMessage::AppendRequest(append_request)) - .await; - assert!(resp.is_ok()); - sk.wal_store.truncate_wal(Lsn(3)).await.unwrap(); // imitate the complete record at 3 %) - assert_eq!(sk.get_last_log_term(), 1); + sk.process_msg(&ProposerAcceptorMessage::AppendRequest(append_request)) + .await + .unwrap(); + assert_eq!(sk.get_last_log_term(), 2); + } + + #[tokio::test] + async fn test_non_consecutive_write() { + let storage = InMemoryState { + persisted_state: test_sk_state(), + }; + let wal_store = DummyWalStore { lsn: Lsn(0) }; + + let mut sk = SafeKeeper::new(TimelineState::new(storage), wal_store, NodeId(0)).unwrap(); + + let pem = ProposerElected { + term: 1, + start_streaming_at: Lsn(1), + term_history: TermHistory(vec![TermLsn { + term: 1, + lsn: Lsn(1), + }]), + timeline_start_lsn: Lsn(1), + }; + sk.process_msg(&ProposerAcceptorMessage::Elected(pem)) + .await + .unwrap(); + + let ar_hdr = AppendRequestHeader { + term: 1, + term_start_lsn: Lsn(3), + begin_lsn: Lsn(1), + end_lsn: Lsn(2), + commit_lsn: Lsn(0), + truncate_lsn: Lsn(0), + proposer_uuid: [0; 16], + }; + let append_request = AppendRequest { + h: ar_hdr.clone(), + wal_data: Bytes::from_static(b"b"), + }; + + // do write ending at 2, it should be ok + sk.process_msg(&ProposerAcceptorMessage::AppendRequest(append_request)) + .await + .unwrap(); + let mut ar_hrd2 = ar_hdr.clone(); + ar_hrd2.begin_lsn = Lsn(4); + ar_hrd2.end_lsn = Lsn(5); + let append_request = AppendRequest { + h: ar_hdr, + wal_data: Bytes::from_static(b"b"), + }; + // and now starting at 4, it must fail + sk.process_msg(&ProposerAcceptorMessage::AppendRequest(append_request)) + .await + .unwrap_err(); } #[test] diff --git a/safekeeper/src/wal_storage.rs b/safekeeper/src/wal_storage.rs index 6fd7c91a68..89c2e98a94 100644 --- a/safekeeper/src/wal_storage.rs +++ b/safekeeper/src/wal_storage.rs @@ -37,6 +37,8 @@ use pq_proto::SystemId; use utils::{id::TenantTimelineId, lsn::Lsn}; pub trait Storage { + // Last written LSN. + fn write_lsn(&self) -> Lsn; /// LSN of last durably stored WAL record. fn flush_lsn(&self) -> Lsn; @@ -329,6 +331,10 @@ impl PhysicalStorage { } impl Storage for PhysicalStorage { + // Last written LSN. + fn write_lsn(&self) -> Lsn { + self.write_lsn + } /// flush_lsn returns LSN of last durably stored WAL record. fn flush_lsn(&self) -> Lsn { self.flush_record_lsn diff --git a/safekeeper/tests/walproposer_sim/safekeeper_disk.rs b/safekeeper/tests/walproposer_sim/safekeeper_disk.rs index 6b31edb1f2..b854754ecf 100644 --- a/safekeeper/tests/walproposer_sim/safekeeper_disk.rs +++ b/safekeeper/tests/walproposer_sim/safekeeper_disk.rs @@ -175,6 +175,10 @@ impl DiskWALStorage { } impl wal_storage::Storage for DiskWALStorage { + // Last written LSN. + fn write_lsn(&self) -> Lsn { + self.write_lsn + } /// LSN of last durably stored WAL record. fn flush_lsn(&self) -> Lsn { self.flush_record_lsn From c4fe6641c1695b1d7c450358af2cec6018fb2359 Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 3 Sep 2024 18:16:49 +0100 Subject: [PATCH 12/45] pageserver: separate metadata and data pages in DatadirModification (#8621) ## Problem Currently, DatadirModification keeps a key-indexed map of all pending writes, even though we (almost) never need to read back dirty pages for anything other than metadata pages (e.g. relation sizes). Related: https://github.com/neondatabase/neon/issues/6345 ## Summary of changes - commit() modifications before ingesting database creation wal records, so that they are guaranteed to be able to get() everything they need directly from the underlying Timeline. - Split dirty pages in DatadirModification into pending_metadata_pages and pending_data_pages. The data ones don't need to be in a key-addressable format, so they just go in a Vec instead. - Special case handling of zero-page writes in DatadirModification, putting them in a map which is flushed on the end of a WAL record. This handles the case where during ingest, we might first write a zero page, and then ingest a postgres write to that page. We used to do this via the key-indexed map of writes, but in this PR we change the data page write path to not bother indexing these by key. My least favorite thing about this PR is that I needed to change the DatadirModification interface to add the on_record_end call. This is not very invasive because there's really only one place we use it, but it changes the object's behaviour from being clearly an aggregation of many records to having some per-record state. I could avoid this by implicitly doing the work when someone calls set_lsn or commit -- I'm open to opinions on whether that's cleaner or dirtier. ## Performance There may be some efficiency improvement here, but the primary motivation is to enable an earlier stage of ingest to operate without access to a Timeline. The `pending_data_pages` part is the "fast path" bulk write data that can in principle be generated without a Timeline, in parallel with other ingest batches, and ultimately on the safekeeper. `test_bulk_insert` on AX102 shows approximately the same results as in the previous PR #8591: ``` ------------------------------ Benchmark results ------------------------------- test_bulk_insert[neon-release-pg16].insert: 23.577 s test_bulk_insert[neon-release-pg16].pageserver_writes: 5,428 MB test_bulk_insert[neon-release-pg16].peak_mem: 637 MB test_bulk_insert[neon-release-pg16].size: 0 MB test_bulk_insert[neon-release-pg16].data_uploaded: 1,922 MB test_bulk_insert[neon-release-pg16].num_files_uploaded: 8 test_bulk_insert[neon-release-pg16].wal_written: 1,382 MB test_bulk_insert[neon-release-pg16].wal_recovery: 18.264 s test_bulk_insert[neon-release-pg16].compaction: 0.052 s ``` --- pageserver/src/import_datadir.rs | 12 +- pageserver/src/pgdatadir_mapping.rs | 228 +++++++++++++----- .../tenant/storage_layer/inmemory_layer.rs | 9 +- .../walreceiver/walreceiver_connection.rs | 64 ++++- pageserver/src/walingest.rs | 42 +++- pageserver/src/walrecord.rs | 24 ++ 6 files changed, 281 insertions(+), 98 deletions(-) diff --git a/pageserver/src/import_datadir.rs b/pageserver/src/import_datadir.rs index ed409d3130..5a0894cd1b 100644 --- a/pageserver/src/import_datadir.rs +++ b/pageserver/src/import_datadir.rs @@ -19,6 +19,7 @@ use crate::metrics::WAL_INGEST; use crate::pgdatadir_mapping::*; use crate::tenant::Timeline; use crate::walingest::WalIngest; +use crate::walrecord::decode_wal_record; use crate::walrecord::DecodedWALRecord; use pageserver_api::reltag::{RelTag, SlruKind}; use postgres_ffi::pg_constants; @@ -310,11 +311,13 @@ async fn import_wal( let mut nrecords = 0; let mut modification = tline.begin_modification(last_lsn); - let mut decoded = DecodedWALRecord::default(); while last_lsn <= endpoint { if let Some((lsn, recdata)) = waldecoder.poll_decode()? { + let mut decoded = DecodedWALRecord::default(); + decode_wal_record(recdata, &mut decoded, tline.pg_version)?; + walingest - .ingest_record(recdata, lsn, &mut modification, &mut decoded, ctx) + .ingest_record(decoded, lsn, &mut modification, ctx) .await?; WAL_INGEST.records_committed.inc(); @@ -449,11 +452,12 @@ pub async fn import_wal_from_tar( waldecoder.feed_bytes(&bytes[offset..]); let mut modification = tline.begin_modification(last_lsn); - let mut decoded = DecodedWALRecord::default(); while last_lsn <= end_lsn { if let Some((lsn, recdata)) = waldecoder.poll_decode()? { + let mut decoded = DecodedWALRecord::default(); + decode_wal_record(recdata, &mut decoded, tline.pg_version)?; walingest - .ingest_record(recdata, lsn, &mut modification, &mut decoded, ctx) + .ingest_record(decoded, lsn, &mut modification, ctx) .await?; modification.commit(ctx).await?; last_lsn = lsn; diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index edcbac970b..c26abca1f7 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -12,7 +12,7 @@ use crate::keyspace::{KeySpace, KeySpaceAccum}; use crate::span::debug_assert_current_span_has_tenant_and_timeline_id_no_shard_id; use crate::walrecord::NeonWalRecord; use crate::{aux_file, repository::*}; -use anyhow::{bail, ensure, Context}; +use anyhow::{ensure, Context}; use bytes::{Buf, Bytes, BytesMut}; use enum_map::Enum; use pageserver_api::key::{ @@ -168,7 +168,9 @@ impl Timeline { DatadirModification { tline: self, pending_lsns: Vec::new(), - pending_updates: HashMap::new(), + pending_metadata_pages: HashMap::new(), + pending_data_pages: Vec::new(), + pending_zero_data_pages: Default::default(), pending_deletions: Vec::new(), pending_nblocks: 0, pending_directory_entries: Vec::new(), @@ -1031,10 +1033,24 @@ pub struct DatadirModification<'a> { // The put-functions add the modifications here, and they are flushed to the // underlying key-value store by the 'finish' function. pending_lsns: Vec, - pending_updates: HashMap>, pending_deletions: Vec<(Range, Lsn)>, pending_nblocks: i64, + /// Metadata writes, indexed by key so that they can be read from not-yet-committed modifications + /// while ingesting subsequent records. See [`Self::is_data_key`] for the definition of 'metadata'. + pending_metadata_pages: HashMap>, + + /// Data writes, ready to be flushed into an ephemeral layer. See [`Self::is_data_key`] for + /// which keys are stored here. + pending_data_pages: Vec<(CompactKey, Lsn, usize, Value)>, + + // Sometimes during ingest, for example when extending a relation, we would like to write a zero page. However, + // if we encounter a write from postgres in the same wal record, we will drop this entry. + // + // Unlike other 'pending' fields, this does not last until the next call to commit(): it is flushed + // at the end of each wal record, and all these writes implicitly are at lsn Self::lsn + pending_zero_data_pages: HashSet, + /// For special "directory" keys that store key-value maps, track the size of the map /// if it was updated in this modification. pending_directory_entries: Vec<(DirectoryKind, usize)>, @@ -1058,6 +1074,10 @@ impl<'a> DatadirModification<'a> { self.pending_bytes } + pub(crate) fn has_dirty_data_pages(&self) -> bool { + (!self.pending_data_pages.is_empty()) || (!self.pending_zero_data_pages.is_empty()) + } + /// Set the current lsn pub(crate) fn set_lsn(&mut self, lsn: Lsn) -> anyhow::Result<()> { ensure!( @@ -1066,6 +1086,10 @@ impl<'a> DatadirModification<'a> { lsn, self.lsn ); + + // If we are advancing LSN, then state from previous wal record should have been flushed. + assert!(self.pending_zero_data_pages.is_empty()); + if lsn > self.lsn { self.pending_lsns.push(self.lsn); self.lsn = lsn; @@ -1073,6 +1097,17 @@ impl<'a> DatadirModification<'a> { Ok(()) } + /// In this context, 'metadata' means keys that are only read by the pageserver internally, and 'data' means + /// keys that represent literal blocks that postgres can read. So data includes relation blocks and + /// SLRU blocks, which are read directly by postgres, and everything else is considered metadata. + /// + /// The distinction is important because data keys are handled on a fast path where dirty writes are + /// not readable until this modification is committed, whereas metadata keys are visible for read + /// via [`Self::get`] as soon as their record has been ingested. + fn is_data_key(key: &Key) -> bool { + key.is_rel_block_key() || key.is_slru_block_key() + } + /// Initialize a completely new repository. /// /// This inserts the directory metadata entries that are assumed to @@ -1180,6 +1215,31 @@ impl<'a> DatadirModification<'a> { Ok(()) } + pub(crate) fn put_rel_page_image_zero(&mut self, rel: RelTag, blknum: BlockNumber) { + self.pending_zero_data_pages + .insert(rel_block_to_key(rel, blknum).to_compact()); + self.pending_bytes += ZERO_PAGE.len(); + } + + pub(crate) fn put_slru_page_image_zero( + &mut self, + kind: SlruKind, + segno: u32, + blknum: BlockNumber, + ) { + self.pending_zero_data_pages + .insert(slru_block_to_key(kind, segno, blknum).to_compact()); + self.pending_bytes += ZERO_PAGE.len(); + } + + /// Call this at the end of each WAL record. + pub(crate) fn on_record_end(&mut self) { + let pending_zero_data_pages = std::mem::take(&mut self.pending_zero_data_pages); + for key in pending_zero_data_pages { + self.put_data(key, Value::Image(ZERO_PAGE.clone())); + } + } + /// Store a relmapper file (pg_filenode.map) in the repository pub async fn put_relmap_file( &mut self, @@ -1778,7 +1838,7 @@ impl<'a> DatadirModification<'a> { /// retains all the metadata, but data pages are flushed. That's again OK /// for bulk import, where you are just loading data pages and won't try to /// modify the same pages twice. - pub async fn flush(&mut self, ctx: &RequestContext) -> anyhow::Result<()> { + pub(crate) async fn flush(&mut self, ctx: &RequestContext) -> anyhow::Result<()> { // Unless we have accumulated a decent amount of changes, it's not worth it // to scan through the pending_updates list. let pending_nblocks = self.pending_nblocks; @@ -1789,31 +1849,11 @@ impl<'a> DatadirModification<'a> { let mut writer = self.tline.writer().await; // Flush relation and SLRU data blocks, keep metadata. - let mut retained_pending_updates = HashMap::<_, Vec<_>>::new(); - for (key, values) in self.pending_updates.drain() { - if !key.is_valid_key_on_write_path() { - bail!( - "the request contains data not supported by pageserver at TimelineWriter::put: {}", key - ); - } - let mut write_batch = Vec::new(); - for (lsn, value_ser_size, value) in values { - if key.is_rel_block_key() || key.is_slru_block_key() { - // This bails out on first error without modifying pending_updates. - // That's Ok, cf this function's doc comment. - write_batch.push((key.to_compact(), lsn, value_ser_size, value)); - } else { - retained_pending_updates.entry(key).or_default().push(( - lsn, - value_ser_size, - value, - )); - } - } - writer.put_batch(write_batch, ctx).await?; - } + let pending_data_pages = std::mem::take(&mut self.pending_data_pages); - self.pending_updates = retained_pending_updates; + // This bails out on first error without modifying pending_updates. + // That's Ok, cf this function's doc comment. + writer.put_batch(pending_data_pages, ctx).await?; self.pending_bytes = 0; if pending_nblocks != 0 { @@ -1834,29 +1874,31 @@ impl<'a> DatadirModification<'a> { /// All the modifications in this atomic update are stamped by the specified LSN. /// pub async fn commit(&mut self, ctx: &RequestContext) -> anyhow::Result<()> { + // Commit should never be called mid-wal-record + assert!(self.pending_zero_data_pages.is_empty()); + let mut writer = self.tline.writer().await; let pending_nblocks = self.pending_nblocks; self.pending_nblocks = 0; - if !self.pending_updates.is_empty() { - // Ordering: the items in this batch do not need to be in any global order, but values for - // a particular Key must be in Lsn order relative to one another. InMemoryLayer relies on - // this to do efficient updates to its index. - let batch: Vec<(CompactKey, Lsn, usize, Value)> = self - .pending_updates + // Ordering: the items in this batch do not need to be in any global order, but values for + // a particular Key must be in Lsn order relative to one another. InMemoryLayer relies on + // this to do efficient updates to its index. + let mut write_batch = std::mem::take(&mut self.pending_data_pages); + + write_batch.extend( + self.pending_metadata_pages .drain() .flat_map(|(key, values)| { - values.into_iter().map(move |(lsn, val_ser_size, value)| { - if !key.is_valid_key_on_write_path() { - bail!("the request contains data not supported by pageserver at TimelineWriter::put: {}", key); - } - Ok((key.to_compact(), lsn, val_ser_size, value)) - }) - }) - .collect::>>()?; + values + .into_iter() + .map(move |(lsn, value_size, value)| (key, lsn, value_size, value)) + }), + ); - writer.put_batch(batch, ctx).await?; + if !write_batch.is_empty() { + writer.put_batch(write_batch, ctx).await?; } if !self.pending_deletions.is_empty() { @@ -1887,33 +1929,58 @@ impl<'a> DatadirModification<'a> { } pub(crate) fn len(&self) -> usize { - self.pending_updates.len() + self.pending_deletions.len() + self.pending_metadata_pages.len() + + self.pending_data_pages.len() + + self.pending_deletions.len() } - // Internal helper functions to batch the modifications - + /// Read a page from the Timeline we are writing to. For metadata pages, this passes through + /// a cache in Self, which makes writes earlier in this modification visible to WAL records later + /// in the modification. + /// + /// For data pages, reads pass directly to the owning Timeline: any ingest code which reads a data + /// page must ensure that the pages they read are already committed in Timeline, for example + /// DB create operations are always preceded by a call to commit(). This is special cased because + /// it's rare: all the 'normal' WAL operations will only read metadata pages such as relation sizes, + /// and not data pages. async fn get(&self, key: Key, ctx: &RequestContext) -> Result { - // Have we already updated the same key? Read the latest pending updated - // version in that case. - // - // Note: we don't check pending_deletions. It is an error to request a - // value that has been removed, deletion only avoids leaking storage. - if let Some(values) = self.pending_updates.get(&key) { - if let Some((_, _, value)) = values.last() { - return if let Value::Image(img) = value { - Ok(img.clone()) - } else { - // Currently, we never need to read back a WAL record that we - // inserted in the same "transaction". All the metadata updates - // work directly with Images, and we never need to read actual - // data pages. We could handle this if we had to, by calling - // the walredo manager, but let's keep it simple for now. - Err(PageReconstructError::Other(anyhow::anyhow!( - "unexpected pending WAL record" - ))) - }; + if !Self::is_data_key(&key) { + // Have we already updated the same key? Read the latest pending updated + // version in that case. + // + // Note: we don't check pending_deletions. It is an error to request a + // value that has been removed, deletion only avoids leaking storage. + if let Some(values) = self.pending_metadata_pages.get(&key.to_compact()) { + if let Some((_, _, value)) = values.last() { + return if let Value::Image(img) = value { + Ok(img.clone()) + } else { + // Currently, we never need to read back a WAL record that we + // inserted in the same "transaction". All the metadata updates + // work directly with Images, and we never need to read actual + // data pages. We could handle this if we had to, by calling + // the walredo manager, but let's keep it simple for now. + Err(PageReconstructError::Other(anyhow::anyhow!( + "unexpected pending WAL record" + ))) + }; + } + } + } else { + // This is an expensive check, so we only do it in debug mode. If reading a data key, + // this key should never be present in pending_data_pages. We ensure this by committing + // modifications before ingesting DB create operations, which are the only kind that reads + // data pages during ingest. + if cfg!(debug_assertions) { + for (dirty_key, _, _, _) in &self.pending_data_pages { + debug_assert!(&key.to_compact() != dirty_key); + } + + debug_assert!(!self.pending_zero_data_pages.contains(&key.to_compact())) } } + + // Metadata page cache miss, or we're reading a data page. let lsn = Lsn::max(self.tline.get_last_record_lsn(), self.lsn); self.tline.get(key, lsn, ctx).await } @@ -1925,11 +1992,40 @@ impl<'a> DatadirModification<'a> { } fn put(&mut self, key: Key, val: Value) { - let values = self.pending_updates.entry(key).or_default(); + if Self::is_data_key(&key) { + self.put_data(key.to_compact(), val) + } else { + self.put_metadata(key.to_compact(), val) + } + } + + fn put_data(&mut self, key: CompactKey, val: Value) { + let val_serialized_size = val.serialized_size().unwrap() as usize; + + // If this page was previously zero'd in the same WalRecord, then drop the previous zero page write. This + // is an optimization that avoids persisting both the zero page generated by us (e.g. during a relation extend), + // and the subsequent postgres-originating write + if self.pending_zero_data_pages.remove(&key) { + self.pending_bytes -= ZERO_PAGE.len(); + } + + self.pending_bytes += val_serialized_size; + self.pending_data_pages + .push((key, self.lsn, val_serialized_size, val)) + } + + fn put_metadata(&mut self, key: CompactKey, val: Value) { + let values = self.pending_metadata_pages.entry(key).or_default(); // Replace the previous value if it exists at the same lsn if let Some((last_lsn, last_value_ser_size, last_value)) = values.last_mut() { if *last_lsn == self.lsn { + // Update the pending_bytes contribution from this entry, and update the serialized size in place + self.pending_bytes -= *last_value_ser_size; *last_value_ser_size = val.serialized_size().unwrap() as usize; + self.pending_bytes += *last_value_ser_size; + + // Use the latest value, this replaces any earlier write to the same (key,lsn), such as much + // have been generated by synthesized zero page writes prior to the first real write to a page. *last_value = val; return; } diff --git a/pageserver/src/tenant/storage_layer/inmemory_layer.rs b/pageserver/src/tenant/storage_layer/inmemory_layer.rs index f31ab4b1e8..2c19e5b19f 100644 --- a/pageserver/src/tenant/storage_layer/inmemory_layer.rs +++ b/pageserver/src/tenant/storage_layer/inmemory_layer.rs @@ -692,8 +692,13 @@ impl InMemoryLayer { let vec_map = inner.index.entry(key).or_default(); let old = vec_map.append_or_update_last(lsn, index_entry).unwrap().0; if old.is_some() { - // We already had an entry for this LSN. That's odd.. - warn!("Key {} at {} already exists", key, lsn); + // This should not break anything, but is unexpected: ingestion code aims to filter out + // multiple writes to the same key at the same LSN. This happens in cases where our + // ingenstion code generates some write like an empty page, and we see a write from postgres + // to the same key in the same wal record. If one such write makes it through, we + // index the most recent write, implicitly ignoring the earlier write. We log a warning + // because this case is unexpected, and we would like tests to fail if this happens. + warn!("Key {} at {} written twice at same LSN", key, lsn); } } diff --git a/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs index 0114473eda..cee259e2e0 100644 --- a/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs +++ b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs @@ -31,7 +31,7 @@ use crate::{ task_mgr::{TaskKind, WALRECEIVER_RUNTIME}, tenant::{debug_assert_current_span_has_tenant_and_timeline_id, Timeline, WalReceiverInfo}, walingest::WalIngest, - walrecord::DecodedWALRecord, + walrecord::{decode_wal_record, DecodedWALRecord}, }; use postgres_backend::is_expected_io_error; use postgres_connection::PgConnectionConfig; @@ -312,10 +312,25 @@ pub(super) async fn handle_walreceiver_connection( waldecoder.feed_bytes(data); { - let mut decoded = DecodedWALRecord::default(); let mut modification = timeline.begin_modification(startlsn); let mut uncommitted_records = 0; let mut filtered_records = 0; + + async fn commit( + modification: &mut DatadirModification<'_>, + uncommitted: &mut u64, + filtered: &mut u64, + ctx: &RequestContext, + ) -> anyhow::Result<()> { + WAL_INGEST + .records_committed + .inc_by(*uncommitted - *filtered); + modification.commit(ctx).await?; + *uncommitted = 0; + *filtered = 0; + Ok(()) + } + while let Some((lsn, recdata)) = waldecoder.poll_decode()? { // It is important to deal with the aligned records as lsn in getPage@LSN is // aligned and can be several bytes bigger. Without this alignment we are @@ -324,9 +339,28 @@ pub(super) async fn handle_walreceiver_connection( return Err(WalReceiverError::Other(anyhow!("LSN not aligned"))); } + // Deserialize WAL record + let mut decoded = DecodedWALRecord::default(); + decode_wal_record(recdata, &mut decoded, modification.tline.pg_version)?; + + if decoded.is_dbase_create_copy(timeline.pg_version) + && uncommitted_records > 0 + { + // Special case: legacy PG database creations operate by reading pages from a 'template' database: + // these are the only kinds of WAL record that require reading data blocks while ingesting. Ensure + // all earlier writes of data blocks are visible by committing any modification in flight. + commit( + &mut modification, + &mut uncommitted_records, + &mut filtered_records, + &ctx, + ) + .await?; + } + // Ingest the records without immediately committing them. let ingested = walingest - .ingest_record(recdata, lsn, &mut modification, &mut decoded, &ctx) + .ingest_record(decoded, lsn, &mut modification, &ctx) .await .with_context(|| format!("could not ingest record at {lsn}"))?; if !ingested { @@ -349,21 +383,25 @@ pub(super) async fn handle_walreceiver_connection( || modification.approx_pending_bytes() > DatadirModification::MAX_PENDING_BYTES { - WAL_INGEST - .records_committed - .inc_by(uncommitted_records - filtered_records); - modification.commit(&ctx).await?; - uncommitted_records = 0; - filtered_records = 0; + commit( + &mut modification, + &mut uncommitted_records, + &mut filtered_records, + &ctx, + ) + .await?; } } // Commit the remaining records. if uncommitted_records > 0 { - WAL_INGEST - .records_committed - .inc_by(uncommitted_records - filtered_records); - modification.commit(&ctx).await?; + commit( + &mut modification, + &mut uncommitted_records, + &mut filtered_records, + &ctx, + ) + .await?; } } diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index 8ccd20adb1..2d3841881b 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -57,6 +57,7 @@ use utils::lsn::Lsn; pub struct WalIngest { shard: ShardIdentity, + pg_version: u32, checkpoint: CheckPoint, checkpoint_modified: bool, warn_ingest_lag: WarnIngestLag, @@ -82,6 +83,7 @@ impl WalIngest { Ok(WalIngest { shard: *timeline.get_shard_identity(), + pg_version: timeline.pg_version, checkpoint, checkpoint_modified: false, warn_ingest_lag: WarnIngestLag { @@ -104,10 +106,9 @@ impl WalIngest { /// pub async fn ingest_record( &mut self, - recdata: Bytes, + decoded: DecodedWALRecord, lsn: Lsn, modification: &mut DatadirModification<'_>, - decoded: &mut DecodedWALRecord, ctx: &RequestContext, ) -> anyhow::Result { WAL_INGEST.records_received.inc(); @@ -115,7 +116,12 @@ impl WalIngest { let prev_len = modification.len(); modification.set_lsn(lsn)?; - decode_wal_record(recdata, decoded, pg_version)?; + + if decoded.is_dbase_create_copy(self.pg_version) { + // Records of this type should always be preceded by a commit(), as they + // rely on reading data pages back from the Timeline. + assert!(!modification.has_dirty_data_pages()); + } let mut buf = decoded.record.clone(); buf.advance(decoded.main_data_offset); @@ -133,11 +139,11 @@ impl WalIngest { pg_constants::RM_HEAP_ID | pg_constants::RM_HEAP2_ID => { // Heap AM records need some special handling, because they modify VM pages // without registering them with the standard mechanism. - self.ingest_heapam_record(&mut buf, modification, decoded, ctx) + self.ingest_heapam_record(&mut buf, modification, &decoded, ctx) .await?; } pg_constants::RM_NEON_ID => { - self.ingest_neonrmgr_record(&mut buf, modification, decoded, ctx) + self.ingest_neonrmgr_record(&mut buf, modification, &decoded, ctx) .await?; } // Handle other special record types @@ -325,7 +331,7 @@ impl WalIngest { } pg_constants::RM_RELMAP_ID => { let xlrec = XlRelmapUpdate::decode(&mut buf); - self.ingest_relmap_page(modification, &xlrec, decoded, ctx) + self.ingest_relmap_page(modification, &xlrec, &decoded, ctx) .await?; } pg_constants::RM_XLOG_ID => { @@ -470,7 +476,7 @@ impl WalIngest { continue; } - self.ingest_decoded_block(modification, lsn, decoded, blk, ctx) + self.ingest_decoded_block(modification, lsn, &decoded, blk, ctx) .await?; } @@ -486,6 +492,8 @@ impl WalIngest { // until commit() is called to flush the data into the repository and update // the latest LSN. + modification.on_record_end(); + Ok(modification.len() > prev_len) } @@ -557,6 +565,7 @@ impl WalIngest { page_set_lsn(&mut image, lsn) } assert_eq!(image.len(), BLCKSZ as usize); + self.put_rel_page_image(modification, rel, blk.blkno, image.freeze(), ctx) .await?; } else { @@ -1195,7 +1204,7 @@ impl WalIngest { if rec.blkno % pg_constants::SLOTS_PER_FSM_PAGE != 0 { // Tail of last remaining FSM page has to be zeroed. // We are not precise here and instead of digging in FSM bitmap format just clear the whole page. - modification.put_rel_page_image(rel, fsm_physical_page_no, ZERO_PAGE.clone())?; + modification.put_rel_page_image_zero(rel, fsm_physical_page_no); fsm_physical_page_no += 1; } let nblocks = get_relsize(modification, rel, ctx).await?; @@ -1217,7 +1226,7 @@ impl WalIngest { if rec.blkno % pg_constants::VM_HEAPBLOCKS_PER_PAGE != 0 { // Tail of last remaining vm page has to be zeroed. // We are not precise here and instead of digging in VM bitmap format just clear the whole page. - modification.put_rel_page_image(rel, vm_page_no, ZERO_PAGE.clone())?; + modification.put_rel_page_image_zero(rel, vm_page_no); vm_page_no += 1; } let nblocks = get_relsize(modification, rel, ctx).await?; @@ -1687,7 +1696,7 @@ impl WalIngest { continue; } - modification.put_rel_page_image(rel, gap_blknum, ZERO_PAGE.clone())?; + modification.put_rel_page_image_zero(rel, gap_blknum); } } Ok(()) @@ -1753,7 +1762,7 @@ impl WalIngest { // fill the gap with zeros for gap_blknum in old_nblocks..blknum { - modification.put_slru_page_image(kind, segno, gap_blknum, ZERO_PAGE.clone())?; + modification.put_slru_page_image_zero(kind, segno, gap_blknum); } } Ok(()) @@ -1827,21 +1836,25 @@ mod tests { walingest .put_rel_page_image(&mut m, TESTREL_A, 0, test_img("foo blk 0 at 2"), &ctx) .await?; + m.on_record_end(); m.commit(&ctx).await?; let mut m = tline.begin_modification(Lsn(0x30)); walingest .put_rel_page_image(&mut m, TESTREL_A, 0, test_img("foo blk 0 at 3"), &ctx) .await?; + m.on_record_end(); m.commit(&ctx).await?; let mut m = tline.begin_modification(Lsn(0x40)); walingest .put_rel_page_image(&mut m, TESTREL_A, 1, test_img("foo blk 1 at 4"), &ctx) .await?; + m.on_record_end(); m.commit(&ctx).await?; let mut m = tline.begin_modification(Lsn(0x50)); walingest .put_rel_page_image(&mut m, TESTREL_A, 2, test_img("foo blk 2 at 5"), &ctx) .await?; + m.on_record_end(); m.commit(&ctx).await?; assert_current_logical_size(&tline, Lsn(0x50)); @@ -1983,6 +1996,7 @@ mod tests { walingest .put_rel_page_image(&mut m, TESTREL_A, 1, test_img("foo blk 1"), &ctx) .await?; + m.on_record_end(); m.commit(&ctx).await?; assert_eq!( tline @@ -2008,6 +2022,7 @@ mod tests { walingest .put_rel_page_image(&mut m, TESTREL_A, 1500, test_img("foo blk 1500"), &ctx) .await?; + m.on_record_end(); m.commit(&ctx).await?; assert_eq!( tline @@ -2409,7 +2424,6 @@ mod tests { .await .unwrap(); let mut modification = tline.begin_modification(startpoint); - let mut decoded = DecodedWALRecord::default(); println!("decoding {} bytes", bytes.len() - xlogoff); // Decode and ingest wal. We process the wal in chunks because @@ -2417,8 +2431,10 @@ mod tests { for chunk in bytes[xlogoff..].chunks(50) { decoder.feed_bytes(chunk); while let Some((lsn, recdata)) = decoder.poll_decode().unwrap() { + let mut decoded = DecodedWALRecord::default(); + decode_wal_record(recdata, &mut decoded, modification.tline.pg_version).unwrap(); walingest - .ingest_record(recdata, lsn, &mut modification, &mut decoded, &ctx) + .ingest_record(decoded, lsn, &mut modification, &ctx) .instrument(span.clone()) .await .unwrap(); diff --git a/pageserver/src/walrecord.rs b/pageserver/src/walrecord.rs index edddcefbe1..0c4d575de8 100644 --- a/pageserver/src/walrecord.rs +++ b/pageserver/src/walrecord.rs @@ -160,6 +160,30 @@ pub struct DecodedWALRecord { pub origin_id: u16, } +impl DecodedWALRecord { + /// Check if this WAL record represents a legacy "copy" database creation, which populates new relations + /// by reading other existing relations' data blocks. This is more complex to apply than new-style database + /// creations which simply include all the desired blocks in the WAL, so we need a helper function to detect this case. + pub(crate) fn is_dbase_create_copy(&self, pg_version: u32) -> bool { + if self.xl_rmid == pg_constants::RM_DBASE_ID { + let info = self.xl_info & pg_constants::XLR_RMGR_INFO_MASK; + match pg_version { + 14 => { + // Postgres 14 database creations are always the legacy kind + info == postgres_ffi::v14::bindings::XLOG_DBASE_CREATE + } + 15 => info == postgres_ffi::v15::bindings::XLOG_DBASE_CREATE_FILE_COPY, + 16 => info == postgres_ffi::v16::bindings::XLOG_DBASE_CREATE_FILE_COPY, + _ => { + panic!("Unsupported postgres version {pg_version}") + } + } + } else { + false + } + } +} + #[repr(C)] #[derive(Debug, Clone, Copy)] pub struct RelFileNode { From 1a874a3e863ac613f52eb0bbfe5e8d83bcfaba55 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Sep 2024 17:31:42 +0000 Subject: [PATCH 13/45] build(deps): bump flask-cors from 4.0.1 to 5.0.0 (#8899) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7db91e51f7..b8ef08b02d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1110,13 +1110,13 @@ dotenv = ["python-dotenv"] [[package]] name = "flask-cors" -version = "4.0.1" +version = "5.0.0" description = "A Flask extension adding a decorator for CORS support" optional = false python-versions = "*" files = [ - {file = "Flask_Cors-4.0.1-py2.py3-none-any.whl", hash = "sha256:f2a704e4458665580c074b714c4627dd5a306b333deb9074d0b1794dfa2fb677"}, - {file = "flask_cors-4.0.1.tar.gz", hash = "sha256:eeb69b342142fdbf4766ad99357a7f3876a2ceb77689dc10ff912aac06c389e4"}, + {file = "Flask_Cors-5.0.0-py2.py3-none-any.whl", hash = "sha256:b9e307d082a9261c100d8fb0ba909eec6a228ed1b60a8315fd85f783d61910bc"}, + {file = "flask_cors-5.0.0.tar.gz", hash = "sha256:5aadb4b950c4e93745034594d9f3ea6591f734bb3662e16e255ffbf5e89c88ef"}, ] [package.dependencies] From 3d9001d83ff54e8bd6a297c3328408323c4e21ff Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." <4198311+skyzh@users.noreply.github.com> Date: Wed, 4 Sep 2024 02:05:06 +0800 Subject: [PATCH 14/45] fix(pageserver): is_archived should be optional (#8902) Set the field to optional, otherwise there will be decode errors when newer version of the storage controller receives the JSON from older version of the pageservers. Signed-off-by: Alex Chi Z --- libs/pageserver_api/src/models.rs | 7 ++++++- pageserver/src/http/routes.rs | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 1d896863df..87e8f8305a 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -716,12 +716,17 @@ pub struct TimelineInfo { pub pg_version: u32, pub state: TimelineState, - pub is_archived: bool, pub walreceiver_status: String, + // ALWAYS add new fields at the end of the struct with `Option` to ensure forward/backward compatibility. + // Backward compatibility: you will get a JSON not containing the newly-added field. + // Forward compatibility: a previous version of the pageserver will receive a JSON. serde::Deserialize does + // not deny unknown fields by default so it's safe to set the field to some value, though it won't be + // read. /// The last aux file policy being used on this timeline pub last_aux_file_policy: Option, + pub is_archived: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 8cf2c99c09..90ae6c5557 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -468,7 +468,7 @@ async fn build_timeline_info_common( pg_version: timeline.pg_version, state, - is_archived, + is_archived: Some(is_archived), walreceiver_status, From ecfa3d9de9eec824800db55f5e9592fe0502c96e Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." <4198311+skyzh@users.noreply.github.com> Date: Wed, 4 Sep 2024 05:39:56 +0800 Subject: [PATCH 15/45] fix(storage-scrubber): wrong trial condition (#8905) ref https://github.com/neondatabase/neon/issues/8872 ## Summary of changes We saw stuck storage scrubber in staging caused by infinite retries. I believe here we should use `min` instead of `max` to avoid getting minutes or hours of retry backoff. Signed-off-by: Alex Chi Z --- storage_scrubber/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/storage_scrubber/src/lib.rs b/storage_scrubber/src/lib.rs index 3c21d2f8cf..3f08cddf50 100644 --- a/storage_scrubber/src/lib.rs +++ b/storage_scrubber/src/lib.rs @@ -422,7 +422,7 @@ fn stream_objects_with_retries<'a>( let yield_err = if err.is_permanent() { true } else { - let backoff_time = 1 << trial.max(5); + let backoff_time = 1 << trial.min(5); tokio::time::sleep(Duration::from_secs(backoff_time)).await; trial += 1; trial == MAX_RETRIES - 1 @@ -473,7 +473,7 @@ async fn list_objects_with_retries( s3_target.delimiter, DisplayErrorContext(e), ); - let backoff_time = 1 << trial.max(5); + let backoff_time = 1 << trial.min(5); tokio::time::sleep(Duration::from_secs(backoff_time)).await; } } @@ -492,7 +492,7 @@ async fn download_object_with_retries( Ok(response) => response, Err(e) => { error!("Failed to download object for key {key}: {e}"); - let backoff_time = 1 << trial.max(5); + let backoff_time = 1 << trial.min(5); tokio::time::sleep(Duration::from_secs(backoff_time)).await; continue; } @@ -508,7 +508,7 @@ async fn download_object_with_retries( } Err(e) => { error!("Failed to stream object body for key {key}: {e}"); - let backoff_time = 1 << trial.max(5); + let backoff_time = 1 << trial.min(5); tokio::time::sleep(Duration::from_secs(backoff_time)).await; } } From 75310fe441b87d399213e365f1364aa9f08aa40d Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Wed, 4 Sep 2024 10:09:41 +0100 Subject: [PATCH 16/45] storcon: make hb interval an argument and speed up tests (#8880) ## Problem Each test might wait for up to 5s in order to HB the pageserver. ## Summary of changes Make the heartbeat interval configurable and use a really tight one for neon local => startup quicker --- control_plane/src/local_env.rs | 7 +++++++ control_plane/src/storage_controller.rs | 2 ++ storage_controller/src/main.rs | 12 ++++++++++-- storage_controller/src/service.rs | 9 ++++++--- test_runner/regress/test_tenants.py | 4 +++- 5 files changed, 28 insertions(+), 6 deletions(-) diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 74caba2b56..5dbc3bcbbc 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -165,6 +165,9 @@ pub struct NeonStorageControllerConf { pub split_threshold: Option, pub max_secondary_lag_bytes: Option, + + #[serde(with = "humantime_serde")] + pub heartbeat_interval: Duration, } impl NeonStorageControllerConf { @@ -172,6 +175,9 @@ impl NeonStorageControllerConf { const DEFAULT_MAX_OFFLINE_INTERVAL: std::time::Duration = std::time::Duration::from_secs(10); const DEFAULT_MAX_WARMING_UP_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30); + + // Very tight heartbeat interval to speed up tests + const DEFAULT_HEARTBEAT_INTERVAL: std::time::Duration = std::time::Duration::from_millis(100); } impl Default for NeonStorageControllerConf { @@ -183,6 +189,7 @@ impl Default for NeonStorageControllerConf { database_url: None, split_threshold: None, max_secondary_lag_bytes: None, + heartbeat_interval: Self::DEFAULT_HEARTBEAT_INTERVAL, } } } diff --git a/control_plane/src/storage_controller.rs b/control_plane/src/storage_controller.rs index 27d8e2de0c..c715d6b789 100644 --- a/control_plane/src/storage_controller.rs +++ b/control_plane/src/storage_controller.rs @@ -437,6 +437,8 @@ impl StorageController { &humantime::Duration::from(self.config.max_offline).to_string(), "--max-warming-up-interval", &humantime::Duration::from(self.config.max_warming_up).to_string(), + "--heartbeat-interval", + &humantime::Duration::from(self.config.heartbeat_interval).to_string(), "--address-for-peers", &address_for_peers.to_string(), ] diff --git a/storage_controller/src/main.rs b/storage_controller/src/main.rs index e3f29b84e7..00e90f4467 100644 --- a/storage_controller/src/main.rs +++ b/storage_controller/src/main.rs @@ -11,8 +11,8 @@ use storage_controller::metrics::preinitialize_metrics; use storage_controller::persistence::Persistence; use storage_controller::service::chaos_injector::ChaosInjector; use storage_controller::service::{ - Config, Service, MAX_OFFLINE_INTERVAL_DEFAULT, MAX_WARMING_UP_INTERVAL_DEFAULT, - RECONCILER_CONCURRENCY_DEFAULT, + Config, Service, HEARTBEAT_INTERVAL_DEFAULT, MAX_OFFLINE_INTERVAL_DEFAULT, + MAX_WARMING_UP_INTERVAL_DEFAULT, RECONCILER_CONCURRENCY_DEFAULT, }; use tokio::signal::unix::SignalKind; use tokio_util::sync::CancellationToken; @@ -104,6 +104,10 @@ struct Cli { // a pageserver #[arg(long)] max_secondary_lag_bytes: Option, + + // Period with which to send heartbeats to registered nodes + #[arg(long)] + heartbeat_interval: Option, } enum StrictMode { @@ -285,6 +289,10 @@ async fn async_main() -> anyhow::Result<()> { split_threshold: args.split_threshold, neon_local_repo_dir: args.neon_local_repo_dir, max_secondary_lag_bytes: args.max_secondary_lag_bytes, + heartbeat_interval: args + .heartbeat_interval + .map(humantime::Duration::into) + .unwrap_or(HEARTBEAT_INTERVAL_DEFAULT), address_for_peers: args.address_for_peers, start_as_candidate: args.start_as_candidate, http_service_port: args.listen.port() as i32, diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 95821827e2..49253cb4e0 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -121,6 +121,9 @@ pub const MAX_OFFLINE_INTERVAL_DEFAULT: Duration = Duration::from_secs(30); /// being handled on the pageserver side. pub const MAX_WARMING_UP_INTERVAL_DEFAULT: Duration = Duration::from_secs(300); +/// How often to send heartbeats to registered nodes? +pub const HEARTBEAT_INTERVAL_DEFAULT: Duration = Duration::from_secs(5); + #[derive(Clone, strum_macros::Display)] enum TenantOperations { Create, @@ -326,6 +329,8 @@ pub struct Config { // upgraded to primary. pub max_secondary_lag_bytes: Option, + pub heartbeat_interval: Duration, + pub address_for_peers: Option, pub start_as_candidate: bool, @@ -909,9 +914,7 @@ impl Service { async fn spawn_heartbeat_driver(&self) { self.startup_complete.clone().wait().await; - const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5); - - let mut interval = tokio::time::interval(HEARTBEAT_INTERVAL); + let mut interval = tokio::time::interval(self.config.heartbeat_interval); while !self.cancel.is_cancelled() { tokio::select! { _ = interval.tick() => { } diff --git a/test_runner/regress/test_tenants.py b/test_runner/regress/test_tenants.py index 0ebf714de0..b63ff7f6bd 100644 --- a/test_runner/regress/test_tenants.py +++ b/test_runner/regress/test_tenants.py @@ -372,8 +372,10 @@ def test_create_churn_during_restart(neon_env_builder: NeonEnvBuilder): tenant_id: TenantId = env.initial_tenant timeline_id = env.initial_timeline - # Multiple creation requests which race will generate this error + # Multiple creation requests which race will generate this error on the pageserver + # and storage controller respectively env.pageserver.allowed_errors.append(".*Conflict: Tenant is already being modified.*") + env.storage_controller.allowed_errors.append(".*Conflict: Tenant is already being modified.*") # Tenant creation requests which arrive out of order will generate complaints about # generation nubmers out of order. From 7a1397cf376cc4169385f6f19c371179396ada5f Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Wed, 4 Sep 2024 13:10:05 +0300 Subject: [PATCH 17/45] storcon: boilerplate to upsert safekeeper records on deploy (#8879) We currently do not record safekeepers in the storage controller database. We want to migrate timelines across safekeepers eventually, so start recording the safekeepers on deploy. Cc: #8698 --- .../2024-08-23-102952_safekeepers/down.sql | 2 + .../2024-08-23-102952_safekeepers/up.sql | 15 ++++ storage_controller/src/http.rs | 57 ++++++++++++ storage_controller/src/persistence.rs | 86 +++++++++++++++++++ storage_controller/src/schema.rs | 14 +++ storage_controller/src/service.rs | 14 +++ test_runner/fixtures/neon_fixtures.py | 23 +++++ .../regress/test_storage_controller.py | 68 ++++++++++++++- 8 files changed, 278 insertions(+), 1 deletion(-) create mode 100644 storage_controller/migrations/2024-08-23-102952_safekeepers/down.sql create mode 100644 storage_controller/migrations/2024-08-23-102952_safekeepers/up.sql diff --git a/storage_controller/migrations/2024-08-23-102952_safekeepers/down.sql b/storage_controller/migrations/2024-08-23-102952_safekeepers/down.sql new file mode 100644 index 0000000000..9dfc750586 --- /dev/null +++ b/storage_controller/migrations/2024-08-23-102952_safekeepers/down.sql @@ -0,0 +1,2 @@ +-- This file should undo anything in `up.sql` +DROP TABLE safekeepers; diff --git a/storage_controller/migrations/2024-08-23-102952_safekeepers/up.sql b/storage_controller/migrations/2024-08-23-102952_safekeepers/up.sql new file mode 100644 index 0000000000..c78716660f --- /dev/null +++ b/storage_controller/migrations/2024-08-23-102952_safekeepers/up.sql @@ -0,0 +1,15 @@ +-- started out as a copy of cplane schema, removed the unnecessary columns. +CREATE TABLE safekeepers ( + -- the surrogate identifier defined by control plane database sequence + id BIGINT PRIMARY KEY, + region_id TEXT NOT NULL, + version BIGINT NOT NULL, + -- the natural id on whatever cloud platform, not needed in storage controller + -- instance_id TEXT UNIQUE NOT NULL, + host TEXT NOT NULL, + port INTEGER NOT NULL, + active BOOLEAN NOT NULL DEFAULT false, + -- projects_count INTEGER NOT NULL DEFAULT 0, + http_port INTEGER NOT NULL, + availability_zone_id TEXT NOT NULL +); diff --git a/storage_controller/src/http.rs b/storage_controller/src/http.rs index d3eb081be4..0fa4f4fd0e 100644 --- a/storage_controller/src/http.rs +++ b/storage_controller/src/http.rs @@ -2,6 +2,7 @@ use crate::metrics::{ HttpRequestLatencyLabelGroup, HttpRequestStatusLabelGroup, PageserverRequestLabelGroup, METRICS_REGISTRY, }; +use crate::persistence::SafekeeperPersistence; use crate::reconciler::ReconcileError; use crate::service::{LeadershipStatus, Service, STARTUP_RECONCILE_TIMEOUT}; use anyhow::Context; @@ -767,6 +768,55 @@ impl From for ApiError { } } +/// Return the safekeeper record by instance id, or 404. +/// +/// Not used by anything except manual testing. +async fn handle_get_safekeeper(req: Request) -> Result, ApiError> { + check_permissions(&req, Scope::Admin)?; + + let id = parse_request_param::(&req, "id")?; + + let state = get_state(&req); + + let res = state.service.get_safekeeper(id).await; + + match res { + Ok(b) => json_response(StatusCode::OK, b), + Err(crate::persistence::DatabaseError::Query(diesel::result::Error::NotFound)) => { + Err(ApiError::NotFound("unknown instance_id".into())) + } + Err(other) => Err(other.into()), + } +} + +/// Used as part of deployment scripts. +/// +/// Assumes information is only relayed to storage controller after first selecting an unique id on +/// control plane database, which means we have an id field in the request and payload. +async fn handle_upsert_safekeeper(mut req: Request) -> Result, ApiError> { + check_permissions(&req, Scope::Admin)?; + + let body = json_request::(&mut req).await?; + let id = parse_request_param::(&req, "id")?; + + if id != body.id { + // it should be repeated + return Err(ApiError::BadRequest(anyhow::anyhow!( + "id mismatch: url={id:?}, body={:?}", + body.id + ))); + } + + let state = get_state(&req); + + state.service.upsert_safekeeper(body).await?; + + Ok(Response::builder() + .status(StatusCode::NO_CONTENT) + .body(Body::empty()) + .unwrap()) +} + /// Common wrapper for request handlers that call into Service and will operate on tenants: they must only /// be allowed to run if Service has finished its initial reconciliation. async fn tenant_service_handler( @@ -1127,6 +1177,13 @@ pub fn make_router( .put("/control/v1/step_down", |r| { named_request_span(r, handle_step_down, RequestName("control_v1_step_down")) }) + .get("/control/v1/safekeeper/:id", |r| { + named_request_span(r, handle_get_safekeeper, RequestName("v1_safekeeper")) + }) + .post("/control/v1/safekeeper/:id", |r| { + // id is in the body + named_request_span(r, handle_upsert_safekeeper, RequestName("v1_safekeeper")) + }) // Tenant operations // The ^/v1/ endpoints act as a "Virtual Pageserver", enabling shard-naive clients to call into // this service to manage tenants that actually consist of many tenant shards, as if they are a single entity. diff --git a/storage_controller/src/persistence.rs b/storage_controller/src/persistence.rs index 6e1c2016ff..d03eb87242 100644 --- a/storage_controller/src/persistence.rs +++ b/storage_controller/src/persistence.rs @@ -938,6 +938,48 @@ impl Persistence { Ok(()) } + + pub(crate) async fn safekeeper_get( + &self, + id: i64, + ) -> Result { + use crate::schema::safekeepers::dsl::{id as id_column, safekeepers}; + self.with_conn(move |conn| -> DatabaseResult { + Ok(safekeepers + .filter(id_column.eq(&id)) + .select(SafekeeperPersistence::as_select()) + .get_result(conn)?) + }) + .await + } + + pub(crate) async fn safekeeper_upsert( + &self, + record: SafekeeperPersistence, + ) -> Result<(), DatabaseError> { + use crate::schema::safekeepers::dsl::*; + + self.with_conn(move |conn| -> DatabaseResult<()> { + let bind = record.as_insert_or_update(); + + let inserted_updated = diesel::insert_into(safekeepers) + .values(&bind) + .on_conflict(id) + .do_update() + .set(&bind) + .execute(conn)?; + + if inserted_updated != 1 { + return Err(DatabaseError::Logical(format!( + "unexpected number of rows ({})", + inserted_updated + ))); + } + + Ok(()) + }) + .await + } } /// Parts of [`crate::tenant_shard::TenantShard`] that are stored durably @@ -1073,3 +1115,47 @@ pub(crate) struct ControllerPersistence { pub(crate) address: String, pub(crate) started_at: chrono::DateTime, } + +#[derive(Serialize, Deserialize, Queryable, Selectable, Eq, PartialEq, Debug, Clone)] +#[diesel(table_name = crate::schema::safekeepers)] +pub(crate) struct SafekeeperPersistence { + pub(crate) id: i64, + pub(crate) region_id: String, + /// 1 is special, it means just created (not currently posted to storcon). + /// Zero or negative is not really expected. + /// Otherwise the number from `release-$(number_of_commits_on_branch)` tag. + pub(crate) version: i64, + pub(crate) host: String, + pub(crate) port: i32, + pub(crate) active: bool, + pub(crate) http_port: i32, + pub(crate) availability_zone_id: String, +} + +impl SafekeeperPersistence { + fn as_insert_or_update(&self) -> InsertUpdateSafekeeper<'_> { + InsertUpdateSafekeeper { + id: self.id, + region_id: &self.region_id, + version: self.version, + host: &self.host, + port: self.port, + active: self.active, + http_port: self.http_port, + availability_zone_id: &self.availability_zone_id, + } + } +} + +#[derive(Insertable, AsChangeset)] +#[diesel(table_name = crate::schema::safekeepers)] +struct InsertUpdateSafekeeper<'a> { + id: i64, + region_id: &'a str, + version: i64, + host: &'a str, + port: i32, + active: bool, + http_port: i32, + availability_zone_id: &'a str, +} diff --git a/storage_controller/src/schema.rs b/storage_controller/src/schema.rs index 1e8379500c..e0f515daea 100644 --- a/storage_controller/src/schema.rs +++ b/storage_controller/src/schema.rs @@ -45,3 +45,17 @@ diesel::table! { } diesel::allow_tables_to_appear_in_same_query!(controllers, metadata_health, nodes, tenant_shards,); + +diesel::table! { + safekeepers { + id -> Int8, + region_id -> Text, + version -> Int8, + instance_id -> Text, + host -> Text, + port -> Int4, + active -> Bool, + http_port -> Int4, + availability_zone_id -> Text, + } +} diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 49253cb4e0..4ccc5c951c 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -6476,4 +6476,18 @@ impl Service { global_observed } + + pub(crate) async fn get_safekeeper( + &self, + id: i64, + ) -> Result { + self.persistence.safekeeper_get(id).await + } + + pub(crate) async fn upsert_safekeeper( + &self, + record: crate::persistence::SafekeeperPersistence, + ) -> Result<(), DatabaseError> { + self.persistence.safekeeper_upsert(record).await + } } diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 8c99408cfb..890538b86a 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2845,6 +2845,29 @@ class NeonStorageController(MetricsGetter, LogUtils): raise AssertionError("unreachable") + def on_safekeeper_deploy(self, id: int, body: dict[str, Any]): + self.request( + "POST", + f"{self.api}/control/v1/safekeeper/{id}", + headers=self.headers(TokenScope.ADMIN), + json=body, + ) + + def get_safekeeper(self, id: int) -> Optional[dict[str, Any]]: + try: + response = self.request( + "GET", + f"{self.api}/control/v1/safekeeper/{id}", + headers=self.headers(TokenScope.ADMIN), + ) + json = response.json() + assert isinstance(json, dict) + return json + except StorageControllerApiException as e: + if e.status_code == 404: + return None + raise e + def __enter__(self) -> "NeonStorageController": return self diff --git a/test_runner/regress/test_storage_controller.py b/test_runner/regress/test_storage_controller.py index 03eb7628be..13f5ec1b4f 100644 --- a/test_runner/regress/test_storage_controller.py +++ b/test_runner/regress/test_storage_controller.py @@ -31,7 +31,7 @@ from fixtures.pageserver.utils import ( remote_storage_delete_key, timeline_delete_wait_completed, ) -from fixtures.pg_version import PgVersion +from fixtures.pg_version import PgVersion, run_only_on_default_postgres from fixtures.port_distributor import PortDistributor from fixtures.remote_storage import RemoteStorageKind, s3_storage from fixtures.storage_controller_proxy import StorageControllerProxy @@ -2330,3 +2330,69 @@ def test_storage_controller_timeline_crud_race(neon_env_builder: NeonEnvBuilder) connect=0, # Disable retries: we want to see the 503 ) ).timeline_create(PgVersion.NOT_SET, tenant_id, create_timeline_id) + + +@run_only_on_default_postgres("this is like a 'unit test' against storcon db") +def test_safekeeper_deployment_time_update(neon_env_builder: NeonEnvBuilder): + env = neon_env_builder.init_configs() + env.start() + + fake_id = 5 + + target = env.storage_controller + + assert target.get_safekeeper(fake_id) is None + + body = { + "active": True, + "id": fake_id, + "created_at": "2023-10-25T09:11:25Z", + "updated_at": "2024-08-28T11:32:43Z", + "region_id": "aws-us-east-2", + "host": "safekeeper-333.us-east-2.aws.neon.build", + "port": 6401, + "http_port": 7676, + "version": 5957, + "availability_zone_id": "us-east-2b", + } + + target.on_safekeeper_deploy(fake_id, body) + + inserted = target.get_safekeeper(fake_id) + assert inserted is not None + assert eq_safekeeper_records(body, inserted) + + # error out if pk is changed (unexpected) + with pytest.raises(StorageControllerApiException) as exc: + different_pk = dict(body) + different_pk["id"] = 4 + assert different_pk["id"] != body["id"] + target.on_safekeeper_deploy(fake_id, different_pk) + assert exc.value.status_code == 400 + + inserted_again = target.get_safekeeper(fake_id) + assert inserted_again is not None + assert eq_safekeeper_records(inserted, inserted_again) + + # the most common case, version goes up: + assert isinstance(body["version"], int) + body["version"] += 1 + target.on_safekeeper_deploy(fake_id, body) + inserted_now = target.get_safekeeper(fake_id) + assert inserted_now is not None + + assert eq_safekeeper_records(body, inserted_now) + + +def eq_safekeeper_records(a: dict[str, Any], b: dict[str, Any]) -> bool: + compared = [dict(a), dict(b)] + + masked_keys = ["created_at", "updated_at"] + + for d in compared: + # keep deleting these in case we are comparing the body as it will be uploaded by real scripts + for key in masked_keys: + if key in d: + del d[key] + + return compared[0] == compared[1] From a046717a2409b5291ad341c1f4d26cb1df1a55bd Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 4 Sep 2024 14:41:51 +0300 Subject: [PATCH 18/45] Fix submodule refs to point to the correct REL_X_STABLE_neon branches (#8910) Commit cfa45ff5ee (PR #8860) updated the vendor/postgres submodules, but didn't use the same commit SHAs that were pushed as the corresponding REL_*_STABLE_neon branches in the postgres repository. The contents were the same, but the REL_*_STABLE_neon branches pointed to squashed versions of the commits, whereas the SHAs used in the submodules referred to the pre-squash revisions. Note: The vendor/postgres-v14 submodule still doesn't match with the tip of REL_14_STABLE_neon branch, because there has been one more commit on that branch since then. That's another confusion which we should fix, but let's do that separately. This commit doesn't change the code that gets built in any way, only changes the submodule references to point to the correct SHAs in the REL_*_STABLE_neon branch histories, rather than some detached commits. --- vendor/postgres-v14 | 2 +- vendor/postgres-v15 | 2 +- vendor/postgres-v16 | 2 +- vendor/revisions.json | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/vendor/postgres-v14 b/vendor/postgres-v14 index 48388a5b59..7602e907ab 160000 --- a/vendor/postgres-v14 +++ b/vendor/postgres-v14 @@ -1 +1 @@ -Subproject commit 48388a5b597c81c09e28c016650a7156b48717a1 +Subproject commit 7602e907ab30f16188bebfd66b8f297c2889d339 diff --git a/vendor/postgres-v15 b/vendor/postgres-v15 index 8aa1ded772..49d5e576a5 160000 --- a/vendor/postgres-v15 +++ b/vendor/postgres-v15 @@ -1 +1 @@ -Subproject commit 8aa1ded7726d416ac8e02600aad387a353478fc7 +Subproject commit 49d5e576a56e4cc59cd6a6a0791b2324b9fa675e diff --git a/vendor/postgres-v16 b/vendor/postgres-v16 index 95132feffe..6e9a4ff624 160000 --- a/vendor/postgres-v16 +++ b/vendor/postgres-v16 @@ -1 +1 @@ -Subproject commit 95132feffe277ce84309d93a42e9aadfd2cb0437 +Subproject commit 6e9a4ff6249ac02b8175054b7b3f7dfb198be48b diff --git a/vendor/revisions.json b/vendor/revisions.json index 319e648488..751b9e8679 100644 --- a/vendor/revisions.json +++ b/vendor/revisions.json @@ -1,14 +1,14 @@ { "v16": [ "16.4", - "95132feffe277ce84309d93a42e9aadfd2cb0437" + "6e9a4ff6249ac02b8175054b7b3f7dfb198be48b" ], "v15": [ "15.8", - "8aa1ded7726d416ac8e02600aad387a353478fc7" + "49d5e576a56e4cc59cd6a6a0791b2324b9fa675e" ], "v14": [ "14.13", - "48388a5b597c81c09e28c016650a7156b48717a1" + "7602e907ab30f16188bebfd66b8f297c2889d339" ] } From 3f43823a9b333140ccf21a55ff1316c351bacd58 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Sep 2024 13:41:10 +0100 Subject: [PATCH 19/45] build(deps): bump cryptography from 42.0.4 to 43.0.1 (#8908) --- poetry.lock | 63 ++++++++++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 34 deletions(-) diff --git a/poetry.lock b/poetry.lock index b8ef08b02d..48943a73e9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -985,43 +985,38 @@ files = [ [[package]] name = "cryptography" -version = "42.0.4" +version = "43.0.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-42.0.4-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:ffc73996c4fca3d2b6c1c8c12bfd3ad00def8621da24f547626bf06441400449"}, - {file = "cryptography-42.0.4-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:db4b65b02f59035037fde0998974d84244a64c3265bdef32a827ab9b63d61b18"}, - {file = "cryptography-42.0.4-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad9c385ba8ee025bb0d856714f71d7840020fe176ae0229de618f14dae7a6e2"}, - {file = "cryptography-42.0.4-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69b22ab6506a3fe483d67d1ed878e1602bdd5912a134e6202c1ec672233241c1"}, - {file = "cryptography-42.0.4-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e09469a2cec88fb7b078e16d4adec594414397e8879a4341c6ace96013463d5b"}, - {file = "cryptography-42.0.4-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3e970a2119507d0b104f0a8e281521ad28fc26f2820687b3436b8c9a5fcf20d1"}, - {file = "cryptography-42.0.4-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:e53dc41cda40b248ebc40b83b31516487f7db95ab8ceac1f042626bc43a2f992"}, - {file = "cryptography-42.0.4-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:c3a5cbc620e1e17009f30dd34cb0d85c987afd21c41a74352d1719be33380885"}, - {file = "cryptography-42.0.4-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6bfadd884e7280df24d26f2186e4e07556a05d37393b0f220a840b083dc6a824"}, - {file = "cryptography-42.0.4-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:01911714117642a3f1792c7f376db572aadadbafcd8d75bb527166009c9f1d1b"}, - {file = "cryptography-42.0.4-cp37-abi3-win32.whl", hash = "sha256:fb0cef872d8193e487fc6bdb08559c3aa41b659a7d9be48b2e10747f47863925"}, - {file = "cryptography-42.0.4-cp37-abi3-win_amd64.whl", hash = "sha256:c1f25b252d2c87088abc8bbc4f1ecbf7c919e05508a7e8628e6875c40bc70923"}, - {file = "cryptography-42.0.4-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:15a1fb843c48b4a604663fa30af60818cd28f895572386e5f9b8a665874c26e7"}, - {file = "cryptography-42.0.4-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1327f280c824ff7885bdeef8578f74690e9079267c1c8bd7dc5cc5aa065ae52"}, - {file = "cryptography-42.0.4-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ffb03d419edcab93b4b19c22ee80c007fb2d708429cecebf1dd3258956a563a"}, - {file = "cryptography-42.0.4-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:1df6fcbf60560d2113b5ed90f072dc0b108d64750d4cbd46a21ec882c7aefce9"}, - {file = "cryptography-42.0.4-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:44a64043f743485925d3bcac548d05df0f9bb445c5fcca6681889c7c3ab12764"}, - {file = "cryptography-42.0.4-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:3c6048f217533d89f2f8f4f0fe3044bf0b2090453b7b73d0b77db47b80af8dff"}, - {file = "cryptography-42.0.4-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:6d0fbe73728c44ca3a241eff9aefe6496ab2656d6e7a4ea2459865f2e8613257"}, - {file = "cryptography-42.0.4-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:887623fe0d70f48ab3f5e4dbf234986b1329a64c066d719432d0698522749929"}, - {file = "cryptography-42.0.4-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce8613beaffc7c14f091497346ef117c1798c202b01153a8cc7b8e2ebaaf41c0"}, - {file = "cryptography-42.0.4-cp39-abi3-win32.whl", hash = "sha256:810bcf151caefc03e51a3d61e53335cd5c7316c0a105cc695f0959f2c638b129"}, - {file = "cryptography-42.0.4-cp39-abi3-win_amd64.whl", hash = "sha256:a0298bdc6e98ca21382afe914c642620370ce0470a01e1bef6dd9b5354c36854"}, - {file = "cryptography-42.0.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5f8907fcf57392cd917892ae83708761c6ff3c37a8e835d7246ff0ad251d9298"}, - {file = "cryptography-42.0.4-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:12d341bd42cdb7d4937b0cabbdf2a94f949413ac4504904d0cdbdce4a22cbf88"}, - {file = "cryptography-42.0.4-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1cdcdbd117681c88d717437ada72bdd5be9de117f96e3f4d50dab3f59fd9ab20"}, - {file = "cryptography-42.0.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:0e89f7b84f421c56e7ff69f11c441ebda73b8a8e6488d322ef71746224c20fce"}, - {file = "cryptography-42.0.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f1e85a178384bf19e36779d91ff35c7617c885da487d689b05c1366f9933ad74"}, - {file = "cryptography-42.0.4-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d2a27aca5597c8a71abbe10209184e1a8e91c1fd470b5070a2ea60cafec35bcd"}, - {file = "cryptography-42.0.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4e36685cb634af55e0677d435d425043967ac2f3790ec652b2b88ad03b85c27b"}, - {file = "cryptography-42.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:f47be41843200f7faec0683ad751e5ef11b9a56a220d57f300376cd8aba81660"}, - {file = "cryptography-42.0.4.tar.gz", hash = "sha256:831a4b37accef30cccd34fcb916a5d7b5be3cbbe27268a02832c3e450aea39cb"}, + {file = "cryptography-43.0.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a"}, + {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042"}, + {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494"}, + {file = "cryptography-43.0.1-cp37-abi3-win32.whl", hash = "sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2"}, + {file = "cryptography-43.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d"}, + {file = "cryptography-43.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1"}, + {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa"}, + {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4"}, + {file = "cryptography-43.0.1-cp39-abi3-win32.whl", hash = "sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47"}, + {file = "cryptography-43.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ea25acb556320250756e53f9e20a4177515f012c9eaea17eb7587a8c4d8ae034"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c1332724be35d23a854994ff0b66530119500b6053d0bd3363265f7e5e77288d"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:fba1007b3ef89946dbbb515aeeb41e30203b004f0b4b00e5e16078b518563289"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5b43d1ea6b378b54a1dc99dd8a2b5be47658fe9a7ce0a58ff0b55f4b43ef2b84"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:88cce104c36870d70c49c7c8fd22885875d950d9ee6ab54df2745f83ba0dc365"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:9d3cdb25fa98afdd3d0892d132b8d7139e2c087da1712041f6b762e4f807cc96"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e710bf40870f4db63c3d7d929aa9e09e4e7ee219e703f949ec4073b4294f6172"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7c05650fe8023c5ed0d46793d4b7d7e6cd9c04e68eabe5b0aeea836e37bdcec2"}, + {file = "cryptography-43.0.1.tar.gz", hash = "sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d"}, ] [package.dependencies] @@ -1034,7 +1029,7 @@ nox = ["nox"] pep8test = ["check-sdist", "click", "mypy", "ruff"] sdist = ["build"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test = ["certifi", "cryptography-vectors (==43.0.1)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] test-randomorder = ["pytest-randomly"] [[package]] From 1a9b54f1d99fb373eddc7f3ff57174031d34c7b6 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 4 Sep 2024 15:00:40 +0100 Subject: [PATCH 20/45] storage controller: read from database in validate API (#8784) ## Problem The initial implementation of the validate API treats the in-memory generations as authoritative. - This is true when only one storage controller is running, but if a rogue controller was running that hadn't been shut down properly, and some pageserver requests were routed to that bad controller, it could incorrectly return valid=true for stale generations. - The generation in the main in-memory map gets out of date while a live migration is in flight, and if the origin location for the migration tries to do some deletions even though it is in AttachedStale (for example because it had already started compaction), these might be wrongly validated + executed. ## Summary of changes - Continue to do the in-memory check: if this returns valid=false it is sufficient to reject requests. - When valid=true, do an additional read from the database to confirm the generation is fresh. - Revise behavior for validation on missing shards: this used to always return valid=true as a convenience for deletions and shard splits, so that pageservers weren't prevented from completing any enqueued deletions for these shards after they're gone. However, this becomes unsafe when we consider split brain scenarios. We could reinstate this in future if we wanted to store some tombstones for deleted shards. - Update test_scrubber_physical_gc to cope with the behavioral change: they must now explicitly flush the deletion queue before splits, to avoid tripping up on deletions that are enqueued at the time of the split (these tests assert "scrubber deletes nothing", which check fails if the split leaves behind some remote objects that are legitimately GC'able) - Add `test_storage_controller_validate_during_migration`, which uses failpoints to create a situation where incorrect generation validation during a live migration could result in a corruption The rate of validate calls for tenants is pretty low: it happens as a consequence deletions from GC and compaction, which are both concurrency-limited on the pageserver side. --- storage_controller/src/http.rs | 2 +- storage_controller/src/persistence.rs | 70 ++++++++++- storage_controller/src/reconciler.rs | 3 + storage_controller/src/service.rs | 91 ++++++++++---- .../regress/test_storage_controller.py | 116 ++++++++++++++++++ test_runner/regress/test_storage_scrubber.py | 11 ++ 6 files changed, 261 insertions(+), 32 deletions(-) diff --git a/storage_controller/src/http.rs b/storage_controller/src/http.rs index 0fa4f4fd0e..32882c201a 100644 --- a/storage_controller/src/http.rs +++ b/storage_controller/src/http.rs @@ -102,7 +102,7 @@ async fn handle_validate(mut req: Request) -> Result, ApiEr let validate_req = json_request::(&mut req).await?; let state = get_state(&req); - json_response(StatusCode::OK, state.service.validate(validate_req)) + json_response(StatusCode::OK, state.service.validate(validate_req).await?) } /// Call into this before attaching a tenant to a pageserver, to acquire a generation number diff --git a/storage_controller/src/persistence.rs b/storage_controller/src/persistence.rs index d03eb87242..e801289752 100644 --- a/storage_controller/src/persistence.rs +++ b/storage_controller/src/persistence.rs @@ -8,6 +8,7 @@ use self::split_state::SplitState; use diesel::pg::PgConnection; use diesel::prelude::*; use diesel::Connection; +use itertools::Itertools; use pageserver_api::controller_api::MetadataHealthRecord; use pageserver_api::controller_api::ShardSchedulingPolicy; use pageserver_api::controller_api::{NodeSchedulingPolicy, PlacementPolicy}; @@ -91,7 +92,8 @@ pub(crate) enum DatabaseOperation { Detach, ReAttach, IncrementGeneration, - PeekGenerations, + TenantGenerations, + ShardGenerations, ListTenantShards, InsertTenantShards, UpdateTenantShard, @@ -544,13 +546,13 @@ impl Persistence { /// If the tenant doesn't exist, an empty vector is returned. /// /// Output is sorted by shard number - pub(crate) async fn peek_generations( + pub(crate) async fn tenant_generations( &self, filter_tenant_id: TenantId, ) -> Result, DatabaseError> { use crate::schema::tenant_shards::dsl::*; let rows = self - .with_measured_conn(DatabaseOperation::PeekGenerations, move |conn| { + .with_measured_conn(DatabaseOperation::TenantGenerations, move |conn| { let result = tenant_shards .filter(tenant_id.eq(filter_tenant_id.to_string())) .select(TenantShardPersistence::as_select()) @@ -572,6 +574,64 @@ impl Persistence { .collect()) } + /// Read the generation number of specific tenant shards + /// + /// Output is unsorted. Output may not include values for all inputs, if they are missing in the database. + pub(crate) async fn shard_generations( + &self, + mut tenant_shard_ids: impl Iterator, + ) -> Result)>, DatabaseError> { + let mut rows = Vec::with_capacity(tenant_shard_ids.size_hint().0); + + // We will chunk our input to avoid composing arbitrarily long `IN` clauses. Typically we are + // called with a single digit number of IDs, but in principle we could be called with tens + // of thousands (all the shards on one pageserver) from the generation validation API. + loop { + // A modest hardcoded chunk size to handle typical cases in a single query but never generate particularly + // large query strings. + let chunk_ids = tenant_shard_ids.by_ref().take(32); + + // Compose a comma separated list of tuples for matching on (tenant_id, shard_number, shard_count) + let in_clause = chunk_ids + .map(|tsid| { + format!( + "('{}', {}, {})", + tsid.tenant_id, tsid.shard_number.0, tsid.shard_count.0 + ) + }) + .join(","); + + // We are done when our iterator gives us nothing to filter on + if in_clause.is_empty() { + break; + } + + let chunk_rows = self + .with_measured_conn(DatabaseOperation::ShardGenerations, move |conn| { + // diesel doesn't support multi-column IN queries, so we compose raw SQL. No escaping is required because + // the inputs are strongly typed and cannot carry any user-supplied raw string content. + let result : Vec = diesel::sql_query( + format!("SELECT * from tenant_shards where (tenant_id, shard_number, shard_count) in ({in_clause});").as_str() + ).load(conn)?; + + Ok(result) + }) + .await?; + rows.extend(chunk_rows.into_iter()) + } + + Ok(rows + .into_iter() + .map(|tsp| { + ( + tsp.get_tenant_shard_id() + .expect("Bad tenant ID in database"), + tsp.generation.map(|g| Generation::new(g as u32)), + ) + }) + .collect()) + } + #[allow(non_local_definitions)] /// For use when updating a persistent property of a tenant, such as its config or placement_policy. /// @@ -983,7 +1043,9 @@ impl Persistence { } /// Parts of [`crate::tenant_shard::TenantShard`] that are stored durably -#[derive(Queryable, Selectable, Insertable, Serialize, Deserialize, Clone, Eq, PartialEq)] +#[derive( + QueryableByName, Queryable, Selectable, Insertable, Serialize, Deserialize, Clone, Eq, PartialEq, +)] #[diesel(table_name = crate::schema::tenant_shards)] pub(crate) struct TenantShardPersistence { #[serde(default)] diff --git a/storage_controller/src/reconciler.rs b/storage_controller/src/reconciler.rs index 102a3124d2..83b7b2b4f2 100644 --- a/storage_controller/src/reconciler.rs +++ b/storage_controller/src/reconciler.rs @@ -17,6 +17,7 @@ use utils::failpoint_support; use utils::generation::Generation; use utils::id::{NodeId, TimelineId}; use utils::lsn::Lsn; +use utils::pausable_failpoint; use utils::sync::gate::GateGuard; use crate::compute_hook::{ComputeHook, NotifyError}; @@ -593,6 +594,8 @@ impl Reconciler { notify_attempts += 1; } + pausable_failpoint!("reconciler-live-migrate-post-notify"); + // Downgrade the origin to secondary. If the tenant's policy is PlacementPolicy::Attached(0), then // this location will be deleted in the general case reconciliation that runs after this. let origin_secondary_conf = build_location_config( diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 4ccc5c951c..90334d10a7 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -1854,37 +1854,74 @@ impl Service { Ok(response) } - pub(crate) fn validate(&self, validate_req: ValidateRequest) -> ValidateResponse { - let locked = self.inner.read().unwrap(); + pub(crate) async fn validate( + &self, + validate_req: ValidateRequest, + ) -> Result { + // Fast in-memory check: we may reject validation on anything that doesn't match our + // in-memory generation for a shard + let in_memory_result = { + let mut in_memory_result = Vec::new(); + let locked = self.inner.read().unwrap(); + for req_tenant in validate_req.tenants { + if let Some(tenant_shard) = locked.tenants.get(&req_tenant.id) { + let valid = tenant_shard.generation == Some(Generation::new(req_tenant.gen)); + tracing::info!( + "handle_validate: {}(gen {}): valid={valid} (latest {:?})", + req_tenant.id, + req_tenant.gen, + tenant_shard.generation + ); + + in_memory_result.push((req_tenant.id, Generation::new(req_tenant.gen), valid)); + } else { + // This is legal: for example during a shard split the pageserver may still + // have deletions in its queue from the old pre-split shard, or after deletion + // of a tenant that was busy with compaction/gc while being deleted. + tracing::info!( + "Refusing deletion validation for missing shard {}", + req_tenant.id + ); + } + } + + in_memory_result + }; + + // Database calls to confirm validity for anything that passed the in-memory check. We must do this + // in case of controller split-brain, where some other controller process might have incremented the generation. + let db_generations = self + .persistence + .shard_generations(in_memory_result.iter().filter_map(|i| { + if i.2 { + Some(&i.0) + } else { + None + } + })) + .await?; + let db_generations = db_generations.into_iter().collect::>(); let mut response = ValidateResponse { tenants: Vec::new(), }; - - for req_tenant in validate_req.tenants { - if let Some(tenant_shard) = locked.tenants.get(&req_tenant.id) { - let valid = tenant_shard.generation == Some(Generation::new(req_tenant.gen)); - tracing::info!( - "handle_validate: {}(gen {}): valid={valid} (latest {:?})", - req_tenant.id, - req_tenant.gen, - tenant_shard.generation - ); - response.tenants.push(ValidateResponseTenant { - id: req_tenant.id, - valid, - }); + for (tenant_shard_id, validate_generation, valid) in in_memory_result.into_iter() { + let valid = if valid { + let db_generation = db_generations.get(&tenant_shard_id); + db_generation == Some(&Some(validate_generation)) } else { - // After tenant deletion, we may approve any validation. This avoids - // spurious warnings on the pageserver if it has pending LSN updates - // at the point a deletion happens. - response.tenants.push(ValidateResponseTenant { - id: req_tenant.id, - valid: true, - }); - } + // If in-memory state says it's invalid, trust that. It's always safe to fail a validation, at worst + // this prevents a pageserver from cleaning up an object in S3. + false + }; + + response.tenants.push(ValidateResponseTenant { + id: tenant_shard_id, + valid, + }) } - response + + Ok(response) } pub(crate) async fn tenant_create( @@ -3179,7 +3216,7 @@ impl Service { // run concurrently with reconciliations, and it is not guaranteed that the node we find here // will still be the latest when we're done: we will check generations again at the end of // this function to handle that. - let generations = self.persistence.peek_generations(tenant_id).await?; + let generations = self.persistence.tenant_generations(tenant_id).await?; if generations .iter() @@ -3236,7 +3273,7 @@ impl Service { // Post-check: are all the generations of all the shards the same as they were initially? This proves that // our remote operation executed on the latest generation and is therefore persistent. { - let latest_generations = self.persistence.peek_generations(tenant_id).await?; + let latest_generations = self.persistence.tenant_generations(tenant_id).await?; if latest_generations .into_iter() .map( diff --git a/test_runner/regress/test_storage_controller.py b/test_runner/regress/test_storage_controller.py index 13f5ec1b4f..8da42294b0 100644 --- a/test_runner/regress/test_storage_controller.py +++ b/test_runner/regress/test_storage_controller.py @@ -2332,6 +2332,122 @@ def test_storage_controller_timeline_crud_race(neon_env_builder: NeonEnvBuilder) ).timeline_create(PgVersion.NOT_SET, tenant_id, create_timeline_id) +def test_storage_controller_validate_during_migration(neon_env_builder: NeonEnvBuilder): + """ + A correctness edge case: while we are live migrating and a shard's generation is + visible to the Reconciler but not to the central Service, the generation validation + API should still prevent stale generations from doing deletions. + """ + neon_env_builder.num_pageservers = 2 + neon_env_builder.enable_pageserver_remote_storage(s3_storage()) + env = neon_env_builder.init_configs() + env.start() + + TENANT_CONF = { + # small checkpointing and compaction targets to ensure we generate many upload operations + "checkpoint_distance": 128 * 1024, + "compaction_threshold": 1, + "compaction_target_size": 128 * 1024, + # disable background compaction and GC. We invoke it manually when we want it to happen. + "gc_period": "0s", + "compaction_period": "0s", + } + + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline + env.neon_cli.create_tenant(tenant_id, timeline_id) + env.storage_controller.pageserver_api().set_tenant_config(tenant_id, TENANT_CONF) + + # Write enough data that a compaction would do some work (deleting some L0s) + workload = Workload(env, tenant_id, timeline_id) + workload.init() + workload.write_rows(64) + for _i in range(0, 2): + workload.churn_rows(64, upload=False) + + # Upload but don't compact + origin_pageserver = env.get_tenant_pageserver(tenant_id) + dest_ps_id = [p.id for p in env.pageservers if p.id != origin_pageserver.id][0] + origin_pageserver.http_client().timeline_checkpoint( + tenant_id, timeline_id, wait_until_uploaded=True, compact=False + ) + + # Start a compaction that will pause on a failpoint. + compaction_failpoint = "before-upload-index-pausable" + origin_pageserver.http_client().configure_failpoints((compaction_failpoint, "pause")) + + # This failpoint can also cause migration code to time out trying to politely flush + # during migrations + origin_pageserver.allowed_errors.append(".*Timed out waiting for flush to remote storage.*") + + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + compact_fut = executor.submit( + origin_pageserver.http_client().timeline_compact, + tenant_id, + timeline_id, + wait_until_uploaded=True, + ) + + # Let the compaction start and then get stuck uploading an index: when we live migrate, the new generation's + # index will be initialized from the pre-compaction index, referencing layers that the compaction will try to delete + def has_hit_compaction_failpoint(): + assert origin_pageserver.log_contains(f"at failpoint {compaction_failpoint}") + + wait_until(10, 1, has_hit_compaction_failpoint) + + # While the compaction is running, start a live migration which will pause long enough for the compaction to sleep, + # after incrementing generation and attaching the new location + migration_failpoint = "reconciler-live-migrate-post-notify" + env.storage_controller.configure_failpoints((migration_failpoint, "pause")) + migrate_fut = executor.submit( + env.storage_controller.tenant_shard_migrate, + TenantShardId(tenant_id, 0, 0), + dest_ps_id, + ) + + def has_hit_migration_failpoint(): + assert env.storage_controller.log_contains(f"at failpoint {migration_failpoint}") + + # Long wait because the migration will have to time out during transition to AttachedStale + # before it reaches this point. The timeout is because the AttachedStale transition includes + # a flush of remote storage, and if the compaction already enqueued an index upload this cannot + # make progress. + wait_until(60, 1, has_hit_migration_failpoint) + + # Origin pageserver has succeeded with compaction before the migration completed. It has done all the writes it wanted to do in its own (stale) generation + origin_pageserver.http_client().configure_failpoints((compaction_failpoint, "off")) + compact_fut.result() + origin_pageserver.http_client().deletion_queue_flush(execute=True) + + # Eventually migration completes + env.storage_controller.configure_failpoints((migration_failpoint, "off")) + migrate_fut.result() + except: + # Always disable 'pause' failpoints, even on failure, to avoid hanging in shutdown + env.storage_controller.configure_failpoints((migration_failpoint, "off")) + origin_pageserver.http_client().configure_failpoints((compaction_failpoint, "off")) + raise + + # Ensure the destination of the migration writes an index, so that if it has corrupt state that is + # visible to the scrubber. + workload.write_rows(1, upload=False) + env.get_pageserver(dest_ps_id).http_client().timeline_checkpoint( + tenant_id, timeline_id, wait_until_uploaded=True, compact=False + ) + + # The destination of the live migration would now have a corrupt index (referencing deleted L0s) if + # the controller had not properly applied validation rules. + healthy, _summary = env.storage_scrubber.scan_metadata() + try: + log.info(f"scrubbed, healthy={healthy}") + assert healthy + except: + # On failures, we want to report them FAIL during the test, not as ERROR during teardown + neon_env_builder.enable_scrub_on_exit = False + raise + + @run_only_on_default_postgres("this is like a 'unit test' against storcon db") def test_safekeeper_deployment_time_update(neon_env_builder: NeonEnvBuilder): env = neon_env_builder.init_configs() diff --git a/test_runner/regress/test_storage_scrubber.py b/test_runner/regress/test_storage_scrubber.py index 292a9a1010..848e214c5e 100644 --- a/test_runner/regress/test_storage_scrubber.py +++ b/test_runner/regress/test_storage_scrubber.py @@ -217,6 +217,13 @@ def test_scrubber_physical_gc_ancestors( workload.init() workload.write_rows(100) + # Issue a deletion queue flush so that the parent shard can't leave behind layers + # that will look like unexpected garbage to the scrubber + for pre_split_shard in env.storage_controller.locate(tenant_id): + env.get_pageserver(pre_split_shard["node_id"]).http_client().deletion_queue_flush( + execute=True + ) + new_shard_count = 4 assert shard_count is None or new_shard_count > shard_count shards = env.storage_controller.tenant_shard_split(tenant_id, shard_count=new_shard_count) @@ -321,6 +328,10 @@ def test_scrubber_physical_gc_timeline_deletion(neon_env_builder: NeonEnvBuilder workload.write_rows(100, upload=False) workload.stop() + # Issue a deletion queue flush so that the parent shard can't leave behind layers + # that will look like unexpected garbage to the scrubber + env.get_tenant_pageserver(tenant_id).http_client().deletion_queue_flush(execute=True) + new_shard_count = 4 shards = env.storage_controller.tenant_shard_split(tenant_id, shard_count=new_shard_count) for shard in shards: From 0205ce184967f4510b6034bf2051a495bf464b44 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 4 Sep 2024 17:41:51 +0300 Subject: [PATCH 21/45] Update submodule reference for vendor/postgres-v14 (#8913) There was a confusion on the REL_14_STABLE_neon branch. PR https://github.com/neondatabase/postgres/pull/471 was merged ot the branch, but the corresponding PRs on the other REL_15_STABLE_neon and REL_16_STABLE_neon branches were not merged. Also, the submodule reference in the neon repository was never updated, so even though the REL_14_STABLE_neon branch contained the commit, it was never used. That PR https://github.com/neondatabase/postgres/pull/471 was a few bricks shy of a load (no tests, some differences between the different branches), so to get us to a good state, revert that change from the REL_14_STABLE_neon branch. This PR in the neon repository updates the submodule reference past two commites on the REL_14_STABLE_neon branch: first the commit from PR https://github.com/neondatabase/postgres/pull/471, and immediately after that the revert of the same commit. This brings us back to square one, but now the submodule reference matches the tip of the REL_14_STABLE_neon branch again. --- vendor/postgres-v14 | 2 +- vendor/revisions.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vendor/postgres-v14 b/vendor/postgres-v14 index 7602e907ab..a317b9b5b9 160000 --- a/vendor/postgres-v14 +++ b/vendor/postgres-v14 @@ -1 +1 @@ -Subproject commit 7602e907ab30f16188bebfd66b8f297c2889d339 +Subproject commit a317b9b5b96978b49e78986697f3dd80d06f99a7 diff --git a/vendor/revisions.json b/vendor/revisions.json index 751b9e8679..e52576e61f 100644 --- a/vendor/revisions.json +++ b/vendor/revisions.json @@ -9,6 +9,6 @@ ], "v14": [ "14.13", - "7602e907ab30f16188bebfd66b8f297c2889d339" + "a317b9b5b96978b49e78986697f3dd80d06f99a7" ] } From 99fa1c36004d710c65a47ffefaf66b4b5c6b4ce1 Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." <4198311+skyzh@users.noreply.github.com> Date: Thu, 5 Sep 2024 04:45:04 +0800 Subject: [PATCH 22/45] fix(pageserver): more information on aux v1 warnings (#8906) Part of https://github.com/neondatabase/neon/issues/8623 ## Summary of changes It seems that we have tenants with aux policy set to v1 but don't have any aux files in the storage. It is still safe to force migrate them without notifying the customers. This patch adds more details to the warning to identify the cases where we have to reach out to the users before retiring aux v1. --------- Signed-off-by: Alex Chi Z --- pageserver/src/pgdatadir_mapping.rs | 10 +++++++--- pageserver/src/tenant/timeline.rs | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index c26abca1f7..d28a214265 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -729,8 +729,12 @@ impl Timeline { let current_policy = self.last_aux_file_policy.load(); match current_policy { Some(AuxFilePolicy::V1) => { - warn!("this timeline is using deprecated aux file policy V1 (policy=V1)"); - self.list_aux_files_v1(lsn, ctx).await + let res = self.list_aux_files_v1(lsn, ctx).await?; + let empty_str = if res.is_empty() { ", empty" } else { "" }; + warn!( + "this timeline is using deprecated aux file policy V1 (policy=v1{empty_str})" + ); + Ok(res) } None => { let res = self.list_aux_files_v1(lsn, ctx).await?; @@ -1657,7 +1661,7 @@ impl<'a> DatadirModification<'a> { if aux_files_key_v1.is_empty() { None } else { - warn!("this timeline is using deprecated aux file policy V1"); + warn!("this timeline is using deprecated aux file policy V1 (detected existing v1 files)"); self.tline.do_switch_aux_policy(AuxFilePolicy::V1)?; Some(AuxFilePolicy::V1) } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 6eadf9a564..3b8f19a6c0 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -2243,7 +2243,7 @@ impl Timeline { }; if aux_file_policy == Some(AuxFilePolicy::V1) { - warn!("this timeline is using deprecated aux file policy V1"); + warn!("this timeline is using deprecated aux file policy V1 (when loading the timeline)"); } result.repartition_threshold = From 708322ce3c0d55bcee5ee9e3632ecfb8c37415f5 Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Thu, 5 Sep 2024 09:56:26 +0100 Subject: [PATCH 23/45] storcon: handle fills including high tput tenants more gracefully (#8865) ## Problem A tenant may ingest a lot of data between being drained for node restart and being moved back in the fill phase. This is expensive and causes the fill to stall. ## Summary of changes We make a tactical change to reduce secondary warm-up time for migrations in fills. --- storage_controller/src/service.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 90334d10a7..ca416095bb 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -6297,9 +6297,13 @@ impl Service { node_id: NodeId, cancel: CancellationToken, ) -> Result<(), OperationError> { - // TODO(vlad): Currently this operates on the assumption that all - // secondaries are warm. This is not always true (e.g. we just migrated the - // tenant). Take that into consideration by checking the secondary status. + const SECONDARY_WARMUP_TIMEOUT: Duration = Duration::from_secs(20); + const SECONDARY_DOWNLOAD_REQUEST_TIMEOUT: Duration = Duration::from_secs(5); + let reconciler_config = ReconcilerConfigBuilder::new() + .secondary_warmup_timeout(SECONDARY_WARMUP_TIMEOUT) + .secondary_download_request_timeout(SECONDARY_DOWNLOAD_REQUEST_TIMEOUT) + .build(); + let mut tids_to_promote = self.fill_node_plan(node_id); let mut waiters = Vec::new(); @@ -6367,9 +6371,11 @@ impl Service { node_id ); - if let Some(waiter) = - self.maybe_reconcile_shard(tenant_shard, nodes) - { + if let Some(waiter) = self.maybe_configured_reconcile_shard( + tenant_shard, + nodes, + reconciler_config, + ) { waiters.push(waiter); } } From 6dfbf49128c4392464d6832ccc2e6bdc390b0b37 Mon Sep 17 00:00:00 2001 From: Folke Behrens Date: Thu, 5 Sep 2024 13:34:27 +0200 Subject: [PATCH 24/45] proxy: don't let one timeout eat entire retry budget (#8924) This reduces the per-request timeout to 10sec while keeping the total retry duration at 1min. Relates: neondatabase/cloud#15944 --- proxy/src/http.rs | 9 ++++++--- proxy/src/usage_metrics.rs | 8 ++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/proxy/src/http.rs b/proxy/src/http.rs index fee634f67f..c77d95f47d 100644 --- a/proxy/src/http.rs +++ b/proxy/src/http.rs @@ -35,14 +35,17 @@ pub fn new_client() -> ClientWithMiddleware { .build() } -pub(crate) fn new_client_with_timeout(default_timout: Duration) -> ClientWithMiddleware { +pub(crate) fn new_client_with_timeout( + request_timeout: Duration, + total_retry_duration: Duration, +) -> ClientWithMiddleware { let timeout_client = reqwest::ClientBuilder::new() - .timeout(default_timout) + .timeout(request_timeout) .build() .expect("Failed to create http client with timeout"); let retry_policy = - ExponentialBackoff::builder().build_with_total_retry_duration(default_timout); + ExponentialBackoff::builder().build_with_total_retry_duration(total_retry_duration); reqwest_middleware::ClientBuilder::new(timeout_client) .with(reqwest_tracing::TracingMiddleware::default()) diff --git a/proxy/src/usage_metrics.rs b/proxy/src/usage_metrics.rs index aa8c7ba319..fd8599bcb3 100644 --- a/proxy/src/usage_metrics.rs +++ b/proxy/src/usage_metrics.rs @@ -33,7 +33,8 @@ use uuid::{NoContext, Timestamp}; const PROXY_IO_BYTES_PER_CLIENT: &str = "proxy_io_bytes_per_client"; -const DEFAULT_HTTP_REPORTING_TIMEOUT: Duration = Duration::from_secs(60); +const HTTP_REPORTING_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); +const HTTP_REPORTING_RETRY_DURATION: Duration = Duration::from_secs(60); /// Key that uniquely identifies the object, this metric describes. /// Currently, endpoint_id is enough, but this may change later, @@ -223,7 +224,10 @@ pub async fn task_main(config: &MetricCollectionConfig) -> anyhow::Result Date: Thu, 5 Sep 2024 14:59:49 +0200 Subject: [PATCH 25/45] refactor(pageserver): rely on serde derive for toml deserialization (#7656) This PR simplifies the pageserver configuration parsing as follows: * introduce the `pageserver_api::config::ConfigToml` type * implement `Default` for `ConfigToml` * use serde derive to do the brain-dead leg-work of processing the toml document * use `serde(default)` to fill in default values * in `pageserver` crate: * use `toml_edit` to deserialize the pageserver.toml string into a `ConfigToml` * `PageServerConfig::parse_and_validate` then * consumes the `ConfigToml` * destructures it exhaustively into its constituent fields * constructs the `PageServerConfig` The rules are: * in `ConfigToml`, use `deny_unknown_fields` everywhere * static default values go in `pageserver_api` * if there cannot be a static default value (e.g. which default IO engine to use, because it depends on the runtime), make the field in `ConfigToml` an `Option` * if runtime-augmentation of a value is needed, do that in `parse_and_validate` * a good example is `virtual_file_io_engine` or `l0_flush`, both of which need to execute code to determine the effective value in `PageServerConf` The benefits: * massive amount of brain-dead repetitive code can be deleted * "unused variable" compile-time errors when removing a config value, due to the exhaustive destructuring in `parse_and_validate` * compile-time errors guide you when adding a new config field Drawbacks: * serde derive is sometimes a bit too magical * `deny_unknown_fields` is easy to miss Future Work / Benefits: * make `neon_local` use `pageserver_api` to construct `ConfigToml` and write it to `pageserver.toml` * This provides more type safety / coompile-time errors than the current approach. ### Refs Fixes #3682 ### Future Work * `remote_storage` deser doesn't reject unknown fields https://github.com/neondatabase/neon/issues/8915 * clean up `libs/pageserver_api/src/config.rs` further * break up into multiple files, at least for tenant config * move `models` as appropriate / refine distinction between config and API models / be explicit about when it's the same * use `pub(crate)` visibility on `mod defaults` to detect stale values --- Cargo.lock | 13 + Cargo.toml | 1 + libs/pageserver_api/Cargo.toml | 10 + libs/pageserver_api/src/config.rs | 527 +++++- libs/pageserver_api/src/models.rs | 71 +- libs/remote_storage/src/config.rs | 25 + libs/utils/src/logging.rs | 12 +- pageserver/Cargo.toml | 3 +- pageserver/benches/bench_ingest.rs | 4 +- pageserver/ctl/src/layer_map_analyzer.rs | 3 +- pageserver/ctl/src/layers.rs | 3 +- pageserver/ctl/src/main.rs | 3 +- pageserver/src/bin/pageserver.rs | 31 +- pageserver/src/config.rs | 1539 +++-------------- pageserver/src/disk_usage_eviction_task.rs | 48 +- pageserver/src/http/routes.rs | 4 +- pageserver/src/l0_flush.rs | 14 +- pageserver/src/statvfs.rs | 28 +- pageserver/src/tenant/config.rs | 196 +-- .../src/tenant/storage_layer/delta_layer.rs | 3 +- .../src/tenant/storage_layer/image_layer.rs | 4 +- .../tenant/storage_layer/inmemory_layer.rs | 2 +- pageserver/src/tenant/tasks.rs | 9 +- pageserver/src/tenant/timeline.rs | 2 +- pageserver/src/tenant/timeline/compaction.rs | 42 +- pageserver/src/tenant/vectored_blob_io.rs | 4 - pageserver/src/virtual_file.rs | 2 +- pageserver/src/virtual_file/io_engine.rs | 11 +- test_runner/fixtures/neon_fixtures.py | 22 +- .../regress/test_pageserver_generations.py | 15 +- test_runner/regress/test_timeline_size.py | 6 +- 31 files changed, 1001 insertions(+), 1656 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5af3ef3804..91917d5351 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2727,6 +2727,12 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "indoc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + [[package]] name = "infer" version = "0.2.3" @@ -3701,6 +3707,7 @@ dependencies = [ "humantime", "humantime-serde", "hyper 0.14.26", + "indoc", "itertools 0.10.5", "md5", "metrics", @@ -3766,6 +3773,7 @@ dependencies = [ "bincode", "byteorder", "bytes", + "camino", "chrono", "const_format", "enum-map", @@ -3773,11 +3781,16 @@ dependencies = [ "humantime", "humantime-serde", "itertools 0.10.5", + "nix 0.27.1", + "postgres_backend", "postgres_ffi", "rand 0.8.5", + "remote_storage", + "reqwest 0.12.4", "serde", "serde_json", "serde_with", + "storage_broker", "strum", "strum_macros", "thiserror", diff --git a/Cargo.toml b/Cargo.toml index fa949f9757..4fea3e8d80 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -103,6 +103,7 @@ humantime-serde = "1.1.1" hyper = "0.14" tokio-tungstenite = "0.20.0" indexmap = "2" +indoc = "2" inotify = "0.10.2" ipnet = "2.9.0" itertools = "0.10" diff --git a/libs/pageserver_api/Cargo.toml b/libs/pageserver_api/Cargo.toml index cb28359ac3..8710904cec 100644 --- a/libs/pageserver_api/Cargo.toml +++ b/libs/pageserver_api/Cargo.toml @@ -4,6 +4,10 @@ version = "0.1.0" edition.workspace = true license.workspace = true +[features] +# See pageserver/Cargo.toml +testing = ["dep:nix"] + [dependencies] serde.workspace = true serde_with.workspace = true @@ -23,6 +27,12 @@ thiserror.workspace = true humantime-serde.workspace = true chrono = { workspace = true, features = ["serde"] } itertools.workspace = true +storage_broker.workspace = true +camino = {workspace = true, features = ["serde1"]} +remote_storage.workspace = true +postgres_backend.workspace = true +nix = {workspace = true, optional = true} +reqwest.workspace = true [dev-dependencies] bincode.workspace = true diff --git a/libs/pageserver_api/src/config.rs b/libs/pageserver_api/src/config.rs index d996a62349..b2662c562a 100644 --- a/libs/pageserver_api/src/config.rs +++ b/libs/pageserver_api/src/config.rs @@ -1,15 +1,28 @@ -use std::collections::HashMap; - -use const_format::formatcp; +use camino::Utf8PathBuf; #[cfg(test)] mod tests; +use const_format::formatcp; pub const DEFAULT_PG_LISTEN_PORT: u16 = 64000; pub const DEFAULT_PG_LISTEN_ADDR: &str = formatcp!("127.0.0.1:{DEFAULT_PG_LISTEN_PORT}"); pub const DEFAULT_HTTP_LISTEN_PORT: u16 = 9898; pub const DEFAULT_HTTP_LISTEN_ADDR: &str = formatcp!("127.0.0.1:{DEFAULT_HTTP_LISTEN_PORT}"); +use postgres_backend::AuthType; +use remote_storage::RemoteStorageConfig; +use serde_with::serde_as; +use std::{ + collections::HashMap, + num::{NonZeroU64, NonZeroUsize}, + str::FromStr, + time::Duration, +}; +use utils::logging::LogFormat; + +use crate::models::ImageCompressionAlgorithm; +use crate::models::LsnLease; + // Certain metadata (e.g. externally-addressable name, AZ) is delivered // as a separate structure. This information is not neeed by the pageserver // itself, it is only used for registering the pageserver with the control @@ -29,3 +42,511 @@ pub struct NodeMetadata { #[serde(flatten)] pub other: HashMap, } + +/// `pageserver.toml` +/// +/// We use serde derive with `#[serde(default)]` to generate a deserializer +/// that fills in the default values for each config field. +/// +/// If there cannot be a static default value because we need to make runtime +/// checks to determine the default, make it an `Option` (which defaults to None). +/// The runtime check should be done in the consuming crate, i.e., `pageserver`. +#[serde_as] +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct ConfigToml { + // types mapped 1:1 into the runtime PageServerConfig type + pub listen_pg_addr: String, + pub listen_http_addr: String, + pub availability_zone: Option, + #[serde(with = "humantime_serde")] + pub wait_lsn_timeout: Duration, + #[serde(with = "humantime_serde")] + pub wal_redo_timeout: Duration, + pub superuser: String, + pub page_cache_size: usize, + pub max_file_descriptors: usize, + pub pg_distrib_dir: Option, + #[serde_as(as = "serde_with::DisplayFromStr")] + pub http_auth_type: AuthType, + #[serde_as(as = "serde_with::DisplayFromStr")] + pub pg_auth_type: AuthType, + pub auth_validation_public_key_path: Option, + pub remote_storage: Option, + pub tenant_config: TenantConfigToml, + #[serde_as(as = "serde_with::DisplayFromStr")] + pub broker_endpoint: storage_broker::Uri, + #[serde(with = "humantime_serde")] + pub broker_keepalive_interval: Duration, + #[serde_as(as = "serde_with::DisplayFromStr")] + pub log_format: LogFormat, + pub concurrent_tenant_warmup: NonZeroUsize, + pub concurrent_tenant_size_logical_size_queries: NonZeroUsize, + #[serde(with = "humantime_serde")] + pub metric_collection_interval: Duration, + pub metric_collection_endpoint: Option, + pub metric_collection_bucket: Option, + #[serde(with = "humantime_serde")] + pub synthetic_size_calculation_interval: Duration, + pub disk_usage_based_eviction: Option, + pub test_remote_failures: u64, + pub ondemand_download_behavior_treat_error_as_warn: bool, + #[serde(with = "humantime_serde")] + pub background_task_maximum_delay: Duration, + pub control_plane_api: Option, + pub control_plane_api_token: Option, + pub control_plane_emergency_mode: bool, + pub heatmap_upload_concurrency: usize, + pub secondary_download_concurrency: usize, + pub virtual_file_io_engine: Option, + pub ingest_batch_size: u64, + pub max_vectored_read_bytes: MaxVectoredReadBytes, + pub image_compression: ImageCompressionAlgorithm, + pub ephemeral_bytes_per_memory_kb: usize, + pub l0_flush: Option, + pub compact_level0_phase1_value_access: CompactL0Phase1ValueAccess, + pub virtual_file_direct_io: crate::models::virtual_file::DirectIoMode, + pub io_buffer_alignment: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DiskUsageEvictionTaskConfig { + pub max_usage_pct: utils::serde_percent::Percent, + pub min_avail_bytes: u64, + #[serde(with = "humantime_serde")] + pub period: Duration, + #[cfg(feature = "testing")] + pub mock_statvfs: Option, + /// Select sorting for evicted layers + #[serde(default)] + pub eviction_order: EvictionOrder, +} + +pub mod statvfs { + pub mod mock { + #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] + #[serde(tag = "type")] + pub enum Behavior { + Success { + blocksize: u64, + total_blocks: u64, + name_filter: Option, + }, + #[cfg(feature = "testing")] + Failure { mocked_error: MockedError }, + } + + #[cfg(feature = "testing")] + #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] + #[allow(clippy::upper_case_acronyms)] + pub enum MockedError { + EIO, + } + + #[cfg(feature = "testing")] + impl From for nix::Error { + fn from(e: MockedError) -> Self { + match e { + MockedError::EIO => nix::Error::EIO, + } + } + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(tag = "type", content = "args")] +pub enum EvictionOrder { + RelativeAccessed { + highest_layer_count_loses_first: bool, + }, +} + +impl Default for EvictionOrder { + fn default() -> Self { + Self::RelativeAccessed { + highest_layer_count_loses_first: true, + } + } +} + +#[derive( + Eq, + PartialEq, + Debug, + Copy, + Clone, + strum_macros::EnumString, + strum_macros::Display, + serde_with::DeserializeFromStr, + serde_with::SerializeDisplay, +)] +#[strum(serialize_all = "kebab-case")] +pub enum GetVectoredImpl { + Sequential, + Vectored, +} + +#[derive( + Eq, + PartialEq, + Debug, + Copy, + Clone, + strum_macros::EnumString, + strum_macros::Display, + serde_with::DeserializeFromStr, + serde_with::SerializeDisplay, +)] +#[strum(serialize_all = "kebab-case")] +pub enum GetImpl { + Legacy, + Vectored, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(transparent)] +pub struct MaxVectoredReadBytes(pub NonZeroUsize); + +#[derive(Debug, PartialEq, Eq, Clone, serde::Deserialize, serde::Serialize)] +#[serde(tag = "mode", rename_all = "kebab-case", deny_unknown_fields)] +pub enum CompactL0Phase1ValueAccess { + /// The old way. + PageCachedBlobIo, + /// The new way. + StreamingKmerge { + /// If set, we run both the old way and the new way, validate that + /// they are identical (=> [`CompactL0BypassPageCacheValidation`]), + /// and if the validation fails, + /// - in tests: fail them with a panic or + /// - in prod, log a rate-limited warning and use the old way's results. + /// + /// If not set, we only run the new way and trust its results. + validate: Option, + }, +} + +/// See [`CompactL0Phase1ValueAccess::StreamingKmerge`]. +#[derive(Debug, PartialEq, Eq, Clone, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum CompactL0BypassPageCacheValidation { + /// Validate that the series of (key, lsn) pairs are the same. + KeyLsn, + /// Validate that the entire output of old and new way is identical. + KeyLsnValue, +} + +impl Default for CompactL0Phase1ValueAccess { + fn default() -> Self { + CompactL0Phase1ValueAccess::StreamingKmerge { + // TODO(https://github.com/neondatabase/neon/issues/8184): change to None once confident + validate: Some(CompactL0BypassPageCacheValidation::KeyLsnValue), + } + } +} + +/// A tenant's calcuated configuration, which is the result of merging a +/// tenant's TenantConfOpt with the global TenantConf from PageServerConf. +/// +/// For storing and transmitting individual tenant's configuration, see +/// TenantConfOpt. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields, default)] +pub struct TenantConfigToml { + // Flush out an inmemory layer, if it's holding WAL older than this + // This puts a backstop on how much WAL needs to be re-digested if the + // page server crashes. + // This parameter actually determines L0 layer file size. + pub checkpoint_distance: u64, + // Inmemory layer is also flushed at least once in checkpoint_timeout to + // eventually upload WAL after activity is stopped. + #[serde(with = "humantime_serde")] + pub checkpoint_timeout: Duration, + // Target file size, when creating image and delta layers. + // This parameter determines L1 layer file size. + pub compaction_target_size: u64, + // How often to check if there's compaction work to be done. + // Duration::ZERO means automatic compaction is disabled. + #[serde(with = "humantime_serde")] + pub compaction_period: Duration, + // Level0 delta layer threshold for compaction. + pub compaction_threshold: usize, + pub compaction_algorithm: crate::models::CompactionAlgorithmSettings, + // Determines how much history is retained, to allow + // branching and read replicas at an older point in time. + // The unit is #of bytes of WAL. + // Page versions older than this are garbage collected away. + pub gc_horizon: u64, + // Interval at which garbage collection is triggered. + // Duration::ZERO means automatic GC is disabled + #[serde(with = "humantime_serde")] + pub gc_period: Duration, + // Delta layer churn threshold to create L1 image layers. + pub image_creation_threshold: usize, + // Determines how much history is retained, to allow + // branching and read replicas at an older point in time. + // The unit is time. + // Page versions older than this are garbage collected away. + #[serde(with = "humantime_serde")] + pub pitr_interval: Duration, + /// Maximum amount of time to wait while opening a connection to receive wal, before erroring. + #[serde(with = "humantime_serde")] + pub walreceiver_connect_timeout: Duration, + /// Considers safekeepers stalled after no WAL updates were received longer than this threshold. + /// A stalled safekeeper will be changed to a newer one when it appears. + #[serde(with = "humantime_serde")] + pub lagging_wal_timeout: Duration, + /// Considers safekeepers lagging when their WAL is behind another safekeeper for more than this threshold. + /// A lagging safekeeper will be changed after `lagging_wal_timeout` time elapses since the last WAL update, + /// to avoid eager reconnects. + pub max_lsn_wal_lag: NonZeroU64, + pub eviction_policy: crate::models::EvictionPolicy, + pub min_resident_size_override: Option, + // See the corresponding metric's help string. + #[serde(with = "humantime_serde")] + pub evictions_low_residence_duration_metric_threshold: Duration, + + /// If non-zero, the period between uploads of a heatmap from attached tenants. This + /// may be disabled if a Tenant will not have secondary locations: only secondary + /// locations will use the heatmap uploaded by attached locations. + #[serde(with = "humantime_serde")] + pub heatmap_period: Duration, + + /// If true then SLRU segments are dowloaded on demand, if false SLRU segments are included in basebackup + pub lazy_slru_download: bool, + + pub timeline_get_throttle: crate::models::ThrottleConfig, + + // How much WAL must be ingested before checking again whether a new image layer is required. + // Expresed in multiples of checkpoint distance. + pub image_layer_creation_check_threshold: u8, + + /// Switch to a new aux file policy. Switching this flag requires the user has not written any aux file into + /// the storage before, and this flag cannot be switched back. Otherwise there will be data corruptions. + /// There is a `last_aux_file_policy` flag which gets persisted in `index_part.json` once the first aux + /// file is written. + pub switch_aux_file_policy: crate::models::AuxFilePolicy, + + /// The length for an explicit LSN lease request. + /// Layers needed to reconstruct pages at LSN will not be GC-ed during this interval. + #[serde(with = "humantime_serde")] + pub lsn_lease_length: Duration, + + /// The length for an implicit LSN lease granted as part of `get_lsn_by_timestamp` request. + /// Layers needed to reconstruct pages at LSN will not be GC-ed during this interval. + #[serde(with = "humantime_serde")] + pub lsn_lease_length_for_ts: Duration, +} + +pub mod defaults { + use crate::models::ImageCompressionAlgorithm; + + pub use storage_broker::DEFAULT_ENDPOINT as BROKER_DEFAULT_ENDPOINT; + + pub const DEFAULT_WAIT_LSN_TIMEOUT: &str = "300 s"; + pub const DEFAULT_WAL_REDO_TIMEOUT: &str = "60 s"; + + pub const DEFAULT_SUPERUSER: &str = "cloud_admin"; + + pub const DEFAULT_PAGE_CACHE_SIZE: usize = 8192; + pub const DEFAULT_MAX_FILE_DESCRIPTORS: usize = 100; + + pub const DEFAULT_LOG_FORMAT: &str = "plain"; + + pub const DEFAULT_CONCURRENT_TENANT_WARMUP: usize = 8; + + pub const DEFAULT_CONCURRENT_TENANT_SIZE_LOGICAL_SIZE_QUERIES: usize = 1; + + pub const DEFAULT_METRIC_COLLECTION_INTERVAL: &str = "10 min"; + pub const DEFAULT_METRIC_COLLECTION_ENDPOINT: Option = None; + pub const DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL: &str = "10 min"; + pub const DEFAULT_BACKGROUND_TASK_MAXIMUM_DELAY: &str = "10s"; + + pub const DEFAULT_HEATMAP_UPLOAD_CONCURRENCY: usize = 8; + pub const DEFAULT_SECONDARY_DOWNLOAD_CONCURRENCY: usize = 1; + + pub const DEFAULT_INGEST_BATCH_SIZE: u64 = 100; + + pub const DEFAULT_MAX_VECTORED_READ_BYTES: usize = 128 * 1024; // 128 KiB + + pub const DEFAULT_IMAGE_COMPRESSION: ImageCompressionAlgorithm = + ImageCompressionAlgorithm::Zstd { level: Some(1) }; + + pub const DEFAULT_VALIDATE_VECTORED_GET: bool = false; + + pub const DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB: usize = 0; + + pub const DEFAULT_IO_BUFFER_ALIGNMENT: usize = 512; +} + +impl Default for ConfigToml { + fn default() -> Self { + use defaults::*; + + Self { + listen_pg_addr: (DEFAULT_PG_LISTEN_ADDR.to_string()), + listen_http_addr: (DEFAULT_HTTP_LISTEN_ADDR.to_string()), + availability_zone: (None), + wait_lsn_timeout: (humantime::parse_duration(DEFAULT_WAIT_LSN_TIMEOUT) + .expect("cannot parse default wait lsn timeout")), + wal_redo_timeout: (humantime::parse_duration(DEFAULT_WAL_REDO_TIMEOUT) + .expect("cannot parse default wal redo timeout")), + superuser: (DEFAULT_SUPERUSER.to_string()), + page_cache_size: (DEFAULT_PAGE_CACHE_SIZE), + max_file_descriptors: (DEFAULT_MAX_FILE_DESCRIPTORS), + pg_distrib_dir: None, // Utf8PathBuf::from("./pg_install"), // TODO: formely, this was std::env::current_dir() + http_auth_type: (AuthType::Trust), + pg_auth_type: (AuthType::Trust), + auth_validation_public_key_path: (None), + remote_storage: None, + broker_endpoint: (storage_broker::DEFAULT_ENDPOINT + .parse() + .expect("failed to parse default broker endpoint")), + broker_keepalive_interval: (humantime::parse_duration( + storage_broker::DEFAULT_KEEPALIVE_INTERVAL, + ) + .expect("cannot parse default keepalive interval")), + log_format: (LogFormat::from_str(DEFAULT_LOG_FORMAT).unwrap()), + + concurrent_tenant_warmup: (NonZeroUsize::new(DEFAULT_CONCURRENT_TENANT_WARMUP) + .expect("Invalid default constant")), + concurrent_tenant_size_logical_size_queries: NonZeroUsize::new(1).unwrap(), + metric_collection_interval: (humantime::parse_duration( + DEFAULT_METRIC_COLLECTION_INTERVAL, + ) + .expect("cannot parse default metric collection interval")), + synthetic_size_calculation_interval: (humantime::parse_duration( + DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL, + ) + .expect("cannot parse default synthetic size calculation interval")), + metric_collection_endpoint: (DEFAULT_METRIC_COLLECTION_ENDPOINT), + + metric_collection_bucket: (None), + + disk_usage_based_eviction: (None), + + test_remote_failures: (0), + + ondemand_download_behavior_treat_error_as_warn: (false), + + background_task_maximum_delay: (humantime::parse_duration( + DEFAULT_BACKGROUND_TASK_MAXIMUM_DELAY, + ) + .unwrap()), + + control_plane_api: (None), + control_plane_api_token: (None), + control_plane_emergency_mode: (false), + + heatmap_upload_concurrency: (DEFAULT_HEATMAP_UPLOAD_CONCURRENCY), + secondary_download_concurrency: (DEFAULT_SECONDARY_DOWNLOAD_CONCURRENCY), + + ingest_batch_size: (DEFAULT_INGEST_BATCH_SIZE), + + virtual_file_io_engine: None, + + max_vectored_read_bytes: (MaxVectoredReadBytes( + NonZeroUsize::new(DEFAULT_MAX_VECTORED_READ_BYTES).unwrap(), + )), + image_compression: (DEFAULT_IMAGE_COMPRESSION), + ephemeral_bytes_per_memory_kb: (DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB), + l0_flush: None, + compact_level0_phase1_value_access: CompactL0Phase1ValueAccess::default(), + virtual_file_direct_io: crate::models::virtual_file::DirectIoMode::default(), + + io_buffer_alignment: DEFAULT_IO_BUFFER_ALIGNMENT, + + tenant_config: TenantConfigToml::default(), + } + } +} + +pub mod tenant_conf_defaults { + + // FIXME: This current value is very low. I would imagine something like 1 GB or 10 GB + // would be more appropriate. But a low value forces the code to be exercised more, + // which is good for now to trigger bugs. + // This parameter actually determines L0 layer file size. + pub const DEFAULT_CHECKPOINT_DISTANCE: u64 = 256 * 1024 * 1024; + pub const DEFAULT_CHECKPOINT_TIMEOUT: &str = "10 m"; + + // FIXME the below configs are only used by legacy algorithm. The new algorithm + // has different parameters. + + // Target file size, when creating image and delta layers. + // This parameter determines L1 layer file size. + pub const DEFAULT_COMPACTION_TARGET_SIZE: u64 = 128 * 1024 * 1024; + + pub const DEFAULT_COMPACTION_PERIOD: &str = "20 s"; + pub const DEFAULT_COMPACTION_THRESHOLD: usize = 10; + pub const DEFAULT_COMPACTION_ALGORITHM: crate::models::CompactionAlgorithm = + crate::models::CompactionAlgorithm::Legacy; + + pub const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024; + + // Large DEFAULT_GC_PERIOD is fine as long as PITR_INTERVAL is larger. + // If there's a need to decrease this value, first make sure that GC + // doesn't hold a layer map write lock for non-trivial operations. + // Relevant: https://github.com/neondatabase/neon/issues/3394 + pub const DEFAULT_GC_PERIOD: &str = "1 hr"; + pub const DEFAULT_IMAGE_CREATION_THRESHOLD: usize = 3; + pub const DEFAULT_PITR_INTERVAL: &str = "7 days"; + pub const DEFAULT_WALRECEIVER_CONNECT_TIMEOUT: &str = "10 seconds"; + pub const DEFAULT_WALRECEIVER_LAGGING_WAL_TIMEOUT: &str = "10 seconds"; + // The default limit on WAL lag should be set to avoid causing disconnects under high throughput + // scenarios: since the broker stats are updated ~1/s, a value of 1GiB should be sufficient for + // throughputs up to 1GiB/s per timeline. + pub const DEFAULT_MAX_WALRECEIVER_LSN_WAL_LAG: u64 = 1024 * 1024 * 1024; + pub const DEFAULT_EVICTIONS_LOW_RESIDENCE_DURATION_METRIC_THRESHOLD: &str = "24 hour"; + // By default ingest enough WAL for two new L0 layers before checking if new image + // image layers should be created. + pub const DEFAULT_IMAGE_LAYER_CREATION_CHECK_THRESHOLD: u8 = 2; + + pub const DEFAULT_INGEST_BATCH_SIZE: u64 = 100; +} + +impl Default for TenantConfigToml { + fn default() -> Self { + use tenant_conf_defaults::*; + Self { + checkpoint_distance: DEFAULT_CHECKPOINT_DISTANCE, + checkpoint_timeout: humantime::parse_duration(DEFAULT_CHECKPOINT_TIMEOUT) + .expect("cannot parse default checkpoint timeout"), + compaction_target_size: DEFAULT_COMPACTION_TARGET_SIZE, + compaction_period: humantime::parse_duration(DEFAULT_COMPACTION_PERIOD) + .expect("cannot parse default compaction period"), + compaction_threshold: DEFAULT_COMPACTION_THRESHOLD, + compaction_algorithm: crate::models::CompactionAlgorithmSettings { + kind: DEFAULT_COMPACTION_ALGORITHM, + }, + gc_horizon: DEFAULT_GC_HORIZON, + gc_period: humantime::parse_duration(DEFAULT_GC_PERIOD) + .expect("cannot parse default gc period"), + image_creation_threshold: DEFAULT_IMAGE_CREATION_THRESHOLD, + pitr_interval: humantime::parse_duration(DEFAULT_PITR_INTERVAL) + .expect("cannot parse default PITR interval"), + walreceiver_connect_timeout: humantime::parse_duration( + DEFAULT_WALRECEIVER_CONNECT_TIMEOUT, + ) + .expect("cannot parse default walreceiver connect timeout"), + lagging_wal_timeout: humantime::parse_duration(DEFAULT_WALRECEIVER_LAGGING_WAL_TIMEOUT) + .expect("cannot parse default walreceiver lagging wal timeout"), + max_lsn_wal_lag: NonZeroU64::new(DEFAULT_MAX_WALRECEIVER_LSN_WAL_LAG) + .expect("cannot parse default max walreceiver Lsn wal lag"), + eviction_policy: crate::models::EvictionPolicy::NoEviction, + min_resident_size_override: None, + evictions_low_residence_duration_metric_threshold: humantime::parse_duration( + DEFAULT_EVICTIONS_LOW_RESIDENCE_DURATION_METRIC_THRESHOLD, + ) + .expect("cannot parse default evictions_low_residence_duration_metric_threshold"), + heatmap_period: Duration::ZERO, + lazy_slru_download: false, + timeline_get_throttle: crate::models::ThrottleConfig::disabled(), + image_layer_creation_check_threshold: DEFAULT_IMAGE_LAYER_CREATION_CHECK_THRESHOLD, + switch_aux_file_policy: crate::models::AuxFilePolicy::default_tenant_config(), + lsn_lease_length: LsnLease::DEFAULT_LENGTH, + lsn_lease_length_for_ts: LsnLease::DEFAULT_LENGTH_FOR_TS, + } + } +} diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 87e8f8305a..d13d04eb1b 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -6,6 +6,7 @@ pub use utilization::PageserverUtilization; use std::{ collections::HashMap, + fmt::Display, io::{BufRead, Read}, num::{NonZeroU32, NonZeroU64, NonZeroUsize}, str::FromStr, @@ -435,7 +436,9 @@ pub enum CompactionAlgorithm { Tiered, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive( + Debug, Clone, Copy, PartialEq, Eq, serde_with::DeserializeFromStr, serde_with::SerializeDisplay, +)] pub enum ImageCompressionAlgorithm { // Disabled for writes, support decompressing during read path Disabled, @@ -470,11 +473,33 @@ impl FromStr for ImageCompressionAlgorithm { } } +impl Display for ImageCompressionAlgorithm { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ImageCompressionAlgorithm::Disabled => write!(f, "disabled"), + ImageCompressionAlgorithm::Zstd { level } => { + if let Some(level) = level { + write!(f, "zstd({})", level) + } else { + write!(f, "zstd") + } + } + } + } +} + #[derive(Eq, PartialEq, Debug, Clone, Serialize, Deserialize)] pub struct CompactionAlgorithmSettings { pub kind: CompactionAlgorithm, } +#[derive(Debug, PartialEq, Eq, Clone, serde::Deserialize, serde::Serialize)] +#[serde(tag = "mode", rename_all = "kebab-case", deny_unknown_fields)] +pub enum L0FlushConfig { + #[serde(rename_all = "snake_case")] + Direct { max_concurrency: NonZeroUsize }, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct EvictionPolicyLayerAccessThreshold { #[serde(with = "humantime_serde")] @@ -1656,21 +1681,33 @@ mod tests { #[test] fn test_image_compression_algorithm_parsing() { use ImageCompressionAlgorithm::*; - assert_eq!( - ImageCompressionAlgorithm::from_str("disabled").unwrap(), - Disabled - ); - assert_eq!( - ImageCompressionAlgorithm::from_str("zstd").unwrap(), - Zstd { level: None } - ); - assert_eq!( - ImageCompressionAlgorithm::from_str("zstd(18)").unwrap(), - Zstd { level: Some(18) } - ); - assert_eq!( - ImageCompressionAlgorithm::from_str("zstd(-3)").unwrap(), - Zstd { level: Some(-3) } - ); + let cases = [ + ("disabled", Disabled), + ("zstd", Zstd { level: None }), + ("zstd(18)", Zstd { level: Some(18) }), + ("zstd(-3)", Zstd { level: Some(-3) }), + ]; + + for (display, expected) in cases { + assert_eq!( + ImageCompressionAlgorithm::from_str(display).unwrap(), + expected, + "parsing works" + ); + assert_eq!(format!("{expected}"), display, "Display FromStr roundtrip"); + + let ser = serde_json::to_string(&expected).expect("serialization"); + assert_eq!( + serde_json::from_str::(&ser).unwrap(), + expected, + "serde roundtrip" + ); + + assert_eq!( + serde_json::Value::String(display.to_string()), + serde_json::to_value(expected).unwrap(), + "Display is the serde serialization" + ); + } } } diff --git a/libs/remote_storage/src/config.rs b/libs/remote_storage/src/config.rs index fa3f2cba58..f819a1572a 100644 --- a/libs/remote_storage/src/config.rs +++ b/libs/remote_storage/src/config.rs @@ -235,6 +235,31 @@ timeout = '5s'"; ); } + #[test] + fn test_storage_class_serde_roundtrip() { + let classes = [ + None, + Some(StorageClass::Standard), + Some(StorageClass::IntelligentTiering), + ]; + for class in classes { + #[derive(Serialize, Deserialize)] + struct Wrapper { + #[serde( + deserialize_with = "deserialize_storage_class", + serialize_with = "serialize_storage_class" + )] + class: Option, + } + let wrapped = Wrapper { + class: class.clone(), + }; + let serialized = serde_json::to_string(&wrapped).unwrap(); + let deserialized: Wrapper = serde_json::from_str(&serialized).unwrap(); + assert_eq!(class, deserialized.class); + } + } + #[test] fn test_azure_parsing() { let toml = "\ diff --git a/libs/utils/src/logging.rs b/libs/utils/src/logging.rs index f7b73dc984..71af43a4da 100644 --- a/libs/utils/src/logging.rs +++ b/libs/utils/src/logging.rs @@ -5,7 +5,9 @@ use metrics::{IntCounter, IntCounterVec}; use once_cell::sync::Lazy; use strum_macros::{EnumString, EnumVariantNames}; -#[derive(EnumString, EnumVariantNames, Eq, PartialEq, Debug, Clone, Copy)] +#[derive( + EnumString, strum_macros::Display, EnumVariantNames, Eq, PartialEq, Debug, Clone, Copy, +)] #[strum(serialize_all = "snake_case")] pub enum LogFormat { Plain, @@ -274,6 +276,14 @@ impl From for SecretString { } } +impl FromStr for SecretString { + type Err = std::convert::Infallible; + + fn from_str(s: &str) -> Result { + Ok(Self(s.to_string())) + } +} + impl std::fmt::Debug for SecretString { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "[SECRET]") diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index 9c02ce3fbc..24373afca3 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -8,7 +8,7 @@ license.workspace = true default = [] # Enables test-only APIs, incuding failpoints. In particular, enables the `fail_point!` macro, # which adds some runtime cost to run tests on outage conditions -testing = ["fail/failpoints"] +testing = ["fail/failpoints", "pageserver_api/testing" ] [dependencies] anyhow.workspace = true @@ -101,6 +101,7 @@ procfs.workspace = true criterion.workspace = true hex-literal.workspace = true tokio = { workspace = true, features = ["process", "sync", "fs", "rt", "io-util", "time", "test-util"] } +indoc.workspace = true [[bench]] name = "bench_layer_map" diff --git a/pageserver/benches/bench_ingest.rs b/pageserver/benches/bench_ingest.rs index 1be4391d81..72cbb6beab 100644 --- a/pageserver/benches/bench_ingest.rs +++ b/pageserver/benches/bench_ingest.rs @@ -4,7 +4,7 @@ use bytes::Bytes; use camino::Utf8PathBuf; use criterion::{criterion_group, criterion_main, Criterion}; use pageserver::{ - config::{defaults::DEFAULT_IO_BUFFER_ALIGNMENT, PageServerConf}, + config::PageServerConf, context::{DownloadBehavior, RequestContext}, l0_flush::{L0FlushConfig, L0FlushGlobalState}, page_cache, @@ -167,7 +167,7 @@ fn criterion_benchmark(c: &mut Criterion) { virtual_file::init( 16384, virtual_file::io_engine_for_bench(), - DEFAULT_IO_BUFFER_ALIGNMENT, + pageserver_api::config::defaults::DEFAULT_IO_BUFFER_ALIGNMENT, ); page_cache::init(conf.page_cache_size); diff --git a/pageserver/ctl/src/layer_map_analyzer.rs b/pageserver/ctl/src/layer_map_analyzer.rs index 8092c203c3..a07107753e 100644 --- a/pageserver/ctl/src/layer_map_analyzer.rs +++ b/pageserver/ctl/src/layer_map_analyzer.rs @@ -4,7 +4,6 @@ use anyhow::Result; use camino::{Utf8Path, Utf8PathBuf}; -use pageserver::config::defaults::DEFAULT_IO_BUFFER_ALIGNMENT; use pageserver::context::{DownloadBehavior, RequestContext}; use pageserver::task_mgr::TaskKind; use pageserver::tenant::{TENANTS_SEGMENT_NAME, TIMELINES_SEGMENT_NAME}; @@ -148,7 +147,7 @@ pub(crate) async fn main(cmd: &AnalyzeLayerMapCmd) -> Result<()> { pageserver::virtual_file::init( 10, virtual_file::api::IoEngineKind::StdFs, - DEFAULT_IO_BUFFER_ALIGNMENT, + pageserver_api::config::defaults::DEFAULT_IO_BUFFER_ALIGNMENT, ); pageserver::page_cache::init(100); diff --git a/pageserver/ctl/src/layers.rs b/pageserver/ctl/src/layers.rs index e0f978eaa2..dd753398e2 100644 --- a/pageserver/ctl/src/layers.rs +++ b/pageserver/ctl/src/layers.rs @@ -3,7 +3,6 @@ use std::path::{Path, PathBuf}; use anyhow::Result; use camino::{Utf8Path, Utf8PathBuf}; use clap::Subcommand; -use pageserver::config::defaults::DEFAULT_IO_BUFFER_ALIGNMENT; use pageserver::context::{DownloadBehavior, RequestContext}; use pageserver::task_mgr::TaskKind; use pageserver::tenant::block_io::BlockCursor; @@ -194,7 +193,7 @@ pub(crate) async fn main(cmd: &LayerCmd) -> Result<()> { pageserver::virtual_file::init( 10, virtual_file::api::IoEngineKind::StdFs, - DEFAULT_IO_BUFFER_ALIGNMENT, + pageserver_api::config::defaults::DEFAULT_IO_BUFFER_ALIGNMENT, ); pageserver::page_cache::init(100); diff --git a/pageserver/ctl/src/main.rs b/pageserver/ctl/src/main.rs index 7a6c7675bb..3b66b0c4aa 100644 --- a/pageserver/ctl/src/main.rs +++ b/pageserver/ctl/src/main.rs @@ -20,14 +20,13 @@ use clap::{Parser, Subcommand}; use index_part::IndexPartCmd; use layers::LayerCmd; use pageserver::{ - config::defaults::DEFAULT_IO_BUFFER_ALIGNMENT, context::{DownloadBehavior, RequestContext}, page_cache, task_mgr::TaskKind, tenant::{dump_layerfile_from_path, metadata::TimelineMetadata}, virtual_file, }; -use pageserver_api::shard::TenantShardId; +use pageserver_api::{config::defaults::DEFAULT_IO_BUFFER_ALIGNMENT, shard::TenantShardId}; use postgres_ffi::ControlFileData; use remote_storage::{RemotePath, RemoteStorageConfig}; use tokio_util::sync::CancellationToken; diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 850bd87b95..2c60e8d7d1 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -5,6 +5,7 @@ use std::env; use std::env::{var, VarError}; use std::io::Read; +use std::str::FromStr; use std::sync::Arc; use std::time::Duration; @@ -223,27 +224,15 @@ fn initialize_config( } }; - let config: toml_edit::Document = match std::fs::File::open(cfg_file_path) { - Ok(mut f) => { - let md = f.metadata().context("stat config file")?; - if md.is_file() { - let mut s = String::new(); - f.read_to_string(&mut s).context("read config file")?; - s.parse().context("parse config file toml")? - } else { - anyhow::bail!("directory entry exists but is not a file: {cfg_file_path}"); - } - } - Err(e) => { - anyhow::bail!("open pageserver config: {e}: {cfg_file_path}"); - } - }; - - debug!("Using pageserver toml: {config}"); - - // Construct the runtime representation - let conf = PageServerConf::parse_and_validate(identity.id, &config, workdir) - .context("Failed to parse pageserver configuration")?; + let config_file_contents = + std::fs::read_to_string(cfg_file_path).context("read config file from filesystem")?; + let config_toml = serde_path_to_error::deserialize( + toml_edit::de::Deserializer::from_str(&config_file_contents) + .context("build toml deserializer")?, + ) + .context("deserialize config toml")?; + let conf = PageServerConf::parse_and_validate(identity.id, config_toml, workdir) + .context("runtime-validation of config toml")?; Ok(Box::leak(Box::new(conf))) } diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index 9e4530ba3c..c159b66905 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -4,11 +4,13 @@ //! file, or on the command line. //! See also `settings.md` for better description on every parameter. -use anyhow::{anyhow, bail, ensure, Context, Result}; -use pageserver_api::{models::ImageCompressionAlgorithm, shard::TenantShardId}; +use anyhow::{bail, ensure, Context}; +use pageserver_api::models::ImageCompressionAlgorithm; +use pageserver_api::{ + config::{DiskUsageEvictionTaskConfig, MaxVectoredReadBytes}, + shard::TenantShardId, +}; use remote_storage::{RemotePath, RemoteStorageConfig}; -use serde::de::IntoDeserializer; -use serde::{self, Deserialize}; use std::env; use storage_broker::Uri; use utils::crashsafe::path_with_suffix_extension; @@ -17,10 +19,8 @@ use utils::logging::SecretString; use once_cell::sync::OnceCell; use reqwest::Url; use std::num::NonZeroUsize; -use std::str::FromStr; use std::sync::Arc; use std::time::Duration; -use toml_edit::{Document, Item}; use camino::{Utf8Path, Utf8PathBuf}; use postgres_backend::AuthType; @@ -29,139 +29,27 @@ use utils::{ logging::LogFormat, }; -use crate::l0_flush::L0FlushConfig; -use crate::tenant::config::TenantConfOpt; use crate::tenant::storage_layer::inmemory_layer::IndexEntry; -use crate::tenant::timeline::compaction::CompactL0Phase1ValueAccess; -use crate::tenant::vectored_blob_io::MaxVectoredReadBytes; use crate::tenant::{TENANTS_SEGMENT_NAME, TIMELINES_SEGMENT_NAME}; -use crate::{disk_usage_eviction_task::DiskUsageEvictionTaskConfig, virtual_file::io_engine}; -use crate::{tenant::config::TenantConf, virtual_file}; +use crate::virtual_file; +use crate::virtual_file::io_engine; use crate::{TENANT_HEATMAP_BASENAME, TENANT_LOCATION_CONFIG_NAME, TIMELINE_DELETE_MARK_SUFFIX}; -use self::defaults::DEFAULT_CONCURRENT_TENANT_WARMUP; - -use self::defaults::DEFAULT_VIRTUAL_FILE_IO_ENGINE; - -pub mod defaults { - use crate::tenant::config::defaults::*; - use const_format::formatcp; - - pub use pageserver_api::config::{ - DEFAULT_HTTP_LISTEN_ADDR, DEFAULT_HTTP_LISTEN_PORT, DEFAULT_PG_LISTEN_ADDR, - DEFAULT_PG_LISTEN_PORT, - }; - pub use storage_broker::DEFAULT_ENDPOINT as BROKER_DEFAULT_ENDPOINT; - - pub const DEFAULT_WAIT_LSN_TIMEOUT: &str = "300 s"; - pub const DEFAULT_WAL_REDO_TIMEOUT: &str = "60 s"; - - pub const DEFAULT_SUPERUSER: &str = "cloud_admin"; - - pub const DEFAULT_PAGE_CACHE_SIZE: usize = 8192; - pub const DEFAULT_MAX_FILE_DESCRIPTORS: usize = 100; - - pub const DEFAULT_LOG_FORMAT: &str = "plain"; - - pub const DEFAULT_CONCURRENT_TENANT_WARMUP: usize = 8; - - pub const DEFAULT_CONCURRENT_TENANT_SIZE_LOGICAL_SIZE_QUERIES: usize = - super::ConfigurableSemaphore::DEFAULT_INITIAL.get(); - - pub const DEFAULT_METRIC_COLLECTION_INTERVAL: &str = "10 min"; - pub const DEFAULT_METRIC_COLLECTION_ENDPOINT: Option = None; - pub const DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL: &str = "10 min"; - pub const DEFAULT_BACKGROUND_TASK_MAXIMUM_DELAY: &str = "10s"; - - pub const DEFAULT_HEATMAP_UPLOAD_CONCURRENCY: usize = 8; - pub const DEFAULT_SECONDARY_DOWNLOAD_CONCURRENCY: usize = 1; - - pub const DEFAULT_INGEST_BATCH_SIZE: u64 = 100; - - #[cfg(target_os = "linux")] - pub const DEFAULT_VIRTUAL_FILE_IO_ENGINE: &str = "tokio-epoll-uring"; - - #[cfg(not(target_os = "linux"))] - pub const DEFAULT_VIRTUAL_FILE_IO_ENGINE: &str = "std-fs"; - - pub const DEFAULT_GET_VECTORED_IMPL: &str = "vectored"; - - pub const DEFAULT_GET_IMPL: &str = "vectored"; - - pub const DEFAULT_MAX_VECTORED_READ_BYTES: usize = 128 * 1024; // 128 KiB - - pub const DEFAULT_IMAGE_COMPRESSION: &str = "zstd(1)"; - - pub const DEFAULT_VALIDATE_VECTORED_GET: bool = false; - - pub const DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB: usize = 0; - - pub const DEFAULT_IO_BUFFER_ALIGNMENT: usize = 512; - - /// - /// Default built-in configuration file. - /// - pub const DEFAULT_CONFIG_FILE: &str = formatcp!( - r#" -# Initial configuration file created by 'pageserver --init' -#listen_pg_addr = '{DEFAULT_PG_LISTEN_ADDR}' -#listen_http_addr = '{DEFAULT_HTTP_LISTEN_ADDR}' - -#wait_lsn_timeout = '{DEFAULT_WAIT_LSN_TIMEOUT}' -#wal_redo_timeout = '{DEFAULT_WAL_REDO_TIMEOUT}' - -#page_cache_size = {DEFAULT_PAGE_CACHE_SIZE} -#max_file_descriptors = {DEFAULT_MAX_FILE_DESCRIPTORS} - -# initial superuser role name to use when creating a new tenant -#initial_superuser_name = '{DEFAULT_SUPERUSER}' - -#broker_endpoint = '{BROKER_DEFAULT_ENDPOINT}' - -#log_format = '{DEFAULT_LOG_FORMAT}' - -#concurrent_tenant_size_logical_size_queries = '{DEFAULT_CONCURRENT_TENANT_SIZE_LOGICAL_SIZE_QUERIES}' -#concurrent_tenant_warmup = '{DEFAULT_CONCURRENT_TENANT_WARMUP}' - -#metric_collection_interval = '{DEFAULT_METRIC_COLLECTION_INTERVAL}' -#synthetic_size_calculation_interval = '{DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL}' - -#disk_usage_based_eviction = {{ max_usage_pct = .., min_avail_bytes = .., period = "10s"}} - -#background_task_maximum_delay = '{DEFAULT_BACKGROUND_TASK_MAXIMUM_DELAY}' - -#ingest_batch_size = {DEFAULT_INGEST_BATCH_SIZE} - -#virtual_file_io_engine = '{DEFAULT_VIRTUAL_FILE_IO_ENGINE}' - -#max_vectored_read_bytes = '{DEFAULT_MAX_VECTORED_READ_BYTES}' - -[tenant_config] -#checkpoint_distance = {DEFAULT_CHECKPOINT_DISTANCE} # in bytes -#checkpoint_timeout = {DEFAULT_CHECKPOINT_TIMEOUT} -#compaction_target_size = {DEFAULT_COMPACTION_TARGET_SIZE} # in bytes -#compaction_period = '{DEFAULT_COMPACTION_PERIOD}' -#compaction_threshold = {DEFAULT_COMPACTION_THRESHOLD} - -#gc_period = '{DEFAULT_GC_PERIOD}' -#gc_horizon = {DEFAULT_GC_HORIZON} -#image_creation_threshold = {DEFAULT_IMAGE_CREATION_THRESHOLD} -#pitr_interval = '{DEFAULT_PITR_INTERVAL}' - -#min_resident_size_override = .. # in bytes -#evictions_low_residence_duration_metric_threshold = '{DEFAULT_EVICTIONS_LOW_RESIDENCE_DURATION_METRIC_THRESHOLD}' - -#heatmap_upload_concurrency = {DEFAULT_HEATMAP_UPLOAD_CONCURRENCY} -#secondary_download_concurrency = {DEFAULT_SECONDARY_DOWNLOAD_CONCURRENCY} - -#ephemeral_bytes_per_memory_kb = {DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB} - -#[remote_storage] - -"# - ); -} - +/// Global state of pageserver. +/// +/// It's mostly immutable configuration, but some semaphores and the +/// like crept in over time and the name stuck. +/// +/// Instantiated by deserializing `pageserver.toml` into [`pageserver_api::config::ConfigToml`] +/// and passing that to [`PageServerConf::parse_and_validate`]. +/// +/// # Adding a New Field +/// +/// 1. Add the field to `pageserver_api::config::ConfigToml`. +/// 2. Fix compiler errors (exhaustive destructuring will guide you). +/// +/// For fields that require additional validation or filling in of defaults at runtime, +/// check for examples in the [`PageServerConf::parse_and_validate`] method. #[derive(Debug, Clone, PartialEq, Eq)] pub struct PageServerConf { // Identifier of that particular pageserver so e g safekeepers @@ -207,7 +95,7 @@ pub struct PageServerConf { pub remote_storage_config: Option, - pub default_tenant_conf: TenantConf, + pub default_tenant_conf: crate::tenant::config::TenantConf, /// Storage broker endpoints to connect to. pub broker_endpoint: Uri, @@ -284,11 +172,11 @@ pub struct PageServerConf { /// Setting this to zero disables limits on total ephemeral layer size. pub ephemeral_bytes_per_memory_kb: usize, - pub l0_flush: L0FlushConfig, + pub l0_flush: crate::l0_flush::L0FlushConfig, /// This flag is temporary and will be removed after gradual rollout. /// See . - pub compact_level0_phase1_value_access: CompactL0Phase1ValueAccess, + pub compact_level0_phase1_value_access: pageserver_api::config::CompactL0Phase1ValueAccess, /// Direct IO settings pub virtual_file_direct_io: virtual_file::DirectIoMode, @@ -304,472 +192,6 @@ pub struct PageServerConf { /// startup code to the connection code through a dozen layers. pub static SAFEKEEPER_AUTH_TOKEN: OnceCell> = OnceCell::new(); -// use dedicated enum for builder to better indicate the intention -// and avoid possible confusion with nested options -#[derive(Clone, Default)] -pub enum BuilderValue { - Set(T), - #[default] - NotSet, -} - -impl BuilderValue { - pub fn ok_or(&self, field_name: &'static str, default: BuilderValue) -> anyhow::Result { - match self { - Self::Set(v) => Ok(v.clone()), - Self::NotSet => match default { - BuilderValue::Set(v) => Ok(v.clone()), - BuilderValue::NotSet => { - anyhow::bail!("missing config value {field_name:?}") - } - }, - } - } -} - -// needed to simplify config construction -#[derive(Default)] -struct PageServerConfigBuilder { - listen_pg_addr: BuilderValue, - - listen_http_addr: BuilderValue, - - availability_zone: BuilderValue>, - - wait_lsn_timeout: BuilderValue, - wal_redo_timeout: BuilderValue, - - superuser: BuilderValue, - - page_cache_size: BuilderValue, - max_file_descriptors: BuilderValue, - - workdir: BuilderValue, - - pg_distrib_dir: BuilderValue, - - http_auth_type: BuilderValue, - pg_auth_type: BuilderValue, - - // - auth_validation_public_key_path: BuilderValue>, - remote_storage_config: BuilderValue>, - - broker_endpoint: BuilderValue, - broker_keepalive_interval: BuilderValue, - - log_format: BuilderValue, - - concurrent_tenant_warmup: BuilderValue, - concurrent_tenant_size_logical_size_queries: BuilderValue, - - metric_collection_interval: BuilderValue, - metric_collection_endpoint: BuilderValue>, - synthetic_size_calculation_interval: BuilderValue, - metric_collection_bucket: BuilderValue>, - - disk_usage_based_eviction: BuilderValue>, - - test_remote_failures: BuilderValue, - - ondemand_download_behavior_treat_error_as_warn: BuilderValue, - - background_task_maximum_delay: BuilderValue, - - control_plane_api: BuilderValue>, - control_plane_api_token: BuilderValue>, - control_plane_emergency_mode: BuilderValue, - - heatmap_upload_concurrency: BuilderValue, - secondary_download_concurrency: BuilderValue, - - ingest_batch_size: BuilderValue, - - virtual_file_io_engine: BuilderValue, - - max_vectored_read_bytes: BuilderValue, - - image_compression: BuilderValue, - - ephemeral_bytes_per_memory_kb: BuilderValue, - - l0_flush: BuilderValue, - - compact_level0_phase1_value_access: BuilderValue, - - virtual_file_direct_io: BuilderValue, - - io_buffer_alignment: BuilderValue, -} - -impl PageServerConfigBuilder { - fn new() -> Self { - Self::default() - } - - #[inline(always)] - fn default_values() -> Self { - use self::BuilderValue::*; - use defaults::*; - Self { - listen_pg_addr: Set(DEFAULT_PG_LISTEN_ADDR.to_string()), - listen_http_addr: Set(DEFAULT_HTTP_LISTEN_ADDR.to_string()), - availability_zone: Set(None), - wait_lsn_timeout: Set(humantime::parse_duration(DEFAULT_WAIT_LSN_TIMEOUT) - .expect("cannot parse default wait lsn timeout")), - wal_redo_timeout: Set(humantime::parse_duration(DEFAULT_WAL_REDO_TIMEOUT) - .expect("cannot parse default wal redo timeout")), - superuser: Set(DEFAULT_SUPERUSER.to_string()), - page_cache_size: Set(DEFAULT_PAGE_CACHE_SIZE), - max_file_descriptors: Set(DEFAULT_MAX_FILE_DESCRIPTORS), - workdir: Set(Utf8PathBuf::new()), - pg_distrib_dir: Set(Utf8PathBuf::from_path_buf( - env::current_dir().expect("cannot access current directory"), - ) - .expect("non-Unicode path") - .join("pg_install")), - http_auth_type: Set(AuthType::Trust), - pg_auth_type: Set(AuthType::Trust), - auth_validation_public_key_path: Set(None), - remote_storage_config: Set(None), - broker_endpoint: Set(storage_broker::DEFAULT_ENDPOINT - .parse() - .expect("failed to parse default broker endpoint")), - broker_keepalive_interval: Set(humantime::parse_duration( - storage_broker::DEFAULT_KEEPALIVE_INTERVAL, - ) - .expect("cannot parse default keepalive interval")), - log_format: Set(LogFormat::from_str(DEFAULT_LOG_FORMAT).unwrap()), - - concurrent_tenant_warmup: Set(NonZeroUsize::new(DEFAULT_CONCURRENT_TENANT_WARMUP) - .expect("Invalid default constant")), - concurrent_tenant_size_logical_size_queries: Set( - ConfigurableSemaphore::DEFAULT_INITIAL, - ), - metric_collection_interval: Set(humantime::parse_duration( - DEFAULT_METRIC_COLLECTION_INTERVAL, - ) - .expect("cannot parse default metric collection interval")), - synthetic_size_calculation_interval: Set(humantime::parse_duration( - DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL, - ) - .expect("cannot parse default synthetic size calculation interval")), - metric_collection_endpoint: Set(DEFAULT_METRIC_COLLECTION_ENDPOINT), - - metric_collection_bucket: Set(None), - - disk_usage_based_eviction: Set(None), - - test_remote_failures: Set(0), - - ondemand_download_behavior_treat_error_as_warn: Set(false), - - background_task_maximum_delay: Set(humantime::parse_duration( - DEFAULT_BACKGROUND_TASK_MAXIMUM_DELAY, - ) - .unwrap()), - - control_plane_api: Set(None), - control_plane_api_token: Set(None), - control_plane_emergency_mode: Set(false), - - heatmap_upload_concurrency: Set(DEFAULT_HEATMAP_UPLOAD_CONCURRENCY), - secondary_download_concurrency: Set(DEFAULT_SECONDARY_DOWNLOAD_CONCURRENCY), - - ingest_batch_size: Set(DEFAULT_INGEST_BATCH_SIZE), - - virtual_file_io_engine: Set(DEFAULT_VIRTUAL_FILE_IO_ENGINE.parse().unwrap()), - - max_vectored_read_bytes: Set(MaxVectoredReadBytes( - NonZeroUsize::new(DEFAULT_MAX_VECTORED_READ_BYTES).unwrap(), - )), - image_compression: Set(DEFAULT_IMAGE_COMPRESSION.parse().unwrap()), - ephemeral_bytes_per_memory_kb: Set(DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB), - l0_flush: Set(L0FlushConfig::default()), - compact_level0_phase1_value_access: Set(CompactL0Phase1ValueAccess::default()), - virtual_file_direct_io: Set(virtual_file::DirectIoMode::default()), - io_buffer_alignment: Set(DEFAULT_IO_BUFFER_ALIGNMENT), - } - } -} - -impl PageServerConfigBuilder { - pub fn listen_pg_addr(&mut self, listen_pg_addr: String) { - self.listen_pg_addr = BuilderValue::Set(listen_pg_addr) - } - - pub fn listen_http_addr(&mut self, listen_http_addr: String) { - self.listen_http_addr = BuilderValue::Set(listen_http_addr) - } - - pub fn availability_zone(&mut self, availability_zone: Option) { - self.availability_zone = BuilderValue::Set(availability_zone) - } - - pub fn wait_lsn_timeout(&mut self, wait_lsn_timeout: Duration) { - self.wait_lsn_timeout = BuilderValue::Set(wait_lsn_timeout) - } - - pub fn wal_redo_timeout(&mut self, wal_redo_timeout: Duration) { - self.wal_redo_timeout = BuilderValue::Set(wal_redo_timeout) - } - - pub fn superuser(&mut self, superuser: String) { - self.superuser = BuilderValue::Set(superuser) - } - - pub fn page_cache_size(&mut self, page_cache_size: usize) { - self.page_cache_size = BuilderValue::Set(page_cache_size) - } - - pub fn max_file_descriptors(&mut self, max_file_descriptors: usize) { - self.max_file_descriptors = BuilderValue::Set(max_file_descriptors) - } - - pub fn workdir(&mut self, workdir: Utf8PathBuf) { - self.workdir = BuilderValue::Set(workdir) - } - - pub fn pg_distrib_dir(&mut self, pg_distrib_dir: Utf8PathBuf) { - self.pg_distrib_dir = BuilderValue::Set(pg_distrib_dir) - } - - pub fn http_auth_type(&mut self, auth_type: AuthType) { - self.http_auth_type = BuilderValue::Set(auth_type) - } - - pub fn pg_auth_type(&mut self, auth_type: AuthType) { - self.pg_auth_type = BuilderValue::Set(auth_type) - } - - pub fn auth_validation_public_key_path( - &mut self, - auth_validation_public_key_path: Option, - ) { - self.auth_validation_public_key_path = BuilderValue::Set(auth_validation_public_key_path) - } - - pub fn remote_storage_config(&mut self, remote_storage_config: Option) { - self.remote_storage_config = BuilderValue::Set(remote_storage_config) - } - - pub fn broker_endpoint(&mut self, broker_endpoint: Uri) { - self.broker_endpoint = BuilderValue::Set(broker_endpoint) - } - - pub fn broker_keepalive_interval(&mut self, broker_keepalive_interval: Duration) { - self.broker_keepalive_interval = BuilderValue::Set(broker_keepalive_interval) - } - - pub fn log_format(&mut self, log_format: LogFormat) { - self.log_format = BuilderValue::Set(log_format) - } - - pub fn concurrent_tenant_warmup(&mut self, u: NonZeroUsize) { - self.concurrent_tenant_warmup = BuilderValue::Set(u); - } - - pub fn concurrent_tenant_size_logical_size_queries(&mut self, u: NonZeroUsize) { - self.concurrent_tenant_size_logical_size_queries = BuilderValue::Set(u); - } - - pub fn metric_collection_interval(&mut self, metric_collection_interval: Duration) { - self.metric_collection_interval = BuilderValue::Set(metric_collection_interval) - } - - pub fn metric_collection_endpoint(&mut self, metric_collection_endpoint: Option) { - self.metric_collection_endpoint = BuilderValue::Set(metric_collection_endpoint) - } - - pub fn metric_collection_bucket( - &mut self, - metric_collection_bucket: Option, - ) { - self.metric_collection_bucket = BuilderValue::Set(metric_collection_bucket) - } - - pub fn synthetic_size_calculation_interval( - &mut self, - synthetic_size_calculation_interval: Duration, - ) { - self.synthetic_size_calculation_interval = - BuilderValue::Set(synthetic_size_calculation_interval) - } - - pub fn test_remote_failures(&mut self, fail_first: u64) { - self.test_remote_failures = BuilderValue::Set(fail_first); - } - - pub fn disk_usage_based_eviction(&mut self, value: Option) { - self.disk_usage_based_eviction = BuilderValue::Set(value); - } - - pub fn ondemand_download_behavior_treat_error_as_warn( - &mut self, - ondemand_download_behavior_treat_error_as_warn: bool, - ) { - self.ondemand_download_behavior_treat_error_as_warn = - BuilderValue::Set(ondemand_download_behavior_treat_error_as_warn); - } - - pub fn background_task_maximum_delay(&mut self, delay: Duration) { - self.background_task_maximum_delay = BuilderValue::Set(delay); - } - - pub fn control_plane_api(&mut self, api: Option) { - self.control_plane_api = BuilderValue::Set(api) - } - - pub fn control_plane_api_token(&mut self, token: Option) { - self.control_plane_api_token = BuilderValue::Set(token) - } - - pub fn control_plane_emergency_mode(&mut self, enabled: bool) { - self.control_plane_emergency_mode = BuilderValue::Set(enabled) - } - - pub fn heatmap_upload_concurrency(&mut self, value: usize) { - self.heatmap_upload_concurrency = BuilderValue::Set(value) - } - - pub fn secondary_download_concurrency(&mut self, value: usize) { - self.secondary_download_concurrency = BuilderValue::Set(value) - } - - pub fn ingest_batch_size(&mut self, ingest_batch_size: u64) { - self.ingest_batch_size = BuilderValue::Set(ingest_batch_size) - } - - pub fn virtual_file_io_engine(&mut self, value: virtual_file::IoEngineKind) { - self.virtual_file_io_engine = BuilderValue::Set(value); - } - - pub fn get_max_vectored_read_bytes(&mut self, value: MaxVectoredReadBytes) { - self.max_vectored_read_bytes = BuilderValue::Set(value); - } - - pub fn get_image_compression(&mut self, value: ImageCompressionAlgorithm) { - self.image_compression = BuilderValue::Set(value); - } - - pub fn get_ephemeral_bytes_per_memory_kb(&mut self, value: usize) { - self.ephemeral_bytes_per_memory_kb = BuilderValue::Set(value); - } - - pub fn l0_flush(&mut self, value: L0FlushConfig) { - self.l0_flush = BuilderValue::Set(value); - } - - pub fn compact_level0_phase1_value_access(&mut self, value: CompactL0Phase1ValueAccess) { - self.compact_level0_phase1_value_access = BuilderValue::Set(value); - } - - pub fn virtual_file_direct_io(&mut self, value: virtual_file::DirectIoMode) { - self.virtual_file_direct_io = BuilderValue::Set(value); - } - - pub fn io_buffer_alignment(&mut self, value: usize) { - self.io_buffer_alignment = BuilderValue::Set(value); - } - - pub fn build(self, id: NodeId) -> anyhow::Result { - let default = Self::default_values(); - - macro_rules! conf { - (USING DEFAULT { $($field:ident,)* } CUSTOM LOGIC { $($custom_field:ident : $custom_value:expr,)* } ) => { - PageServerConf { - $( - $field: self.$field.ok_or(stringify!($field), default.$field)?, - )* - $( - $custom_field: $custom_value, - )* - } - }; - } - - Ok(conf!( - USING DEFAULT - { - listen_pg_addr, - listen_http_addr, - availability_zone, - wait_lsn_timeout, - wal_redo_timeout, - superuser, - page_cache_size, - max_file_descriptors, - workdir, - pg_distrib_dir, - http_auth_type, - pg_auth_type, - auth_validation_public_key_path, - remote_storage_config, - broker_endpoint, - broker_keepalive_interval, - log_format, - metric_collection_interval, - metric_collection_endpoint, - metric_collection_bucket, - synthetic_size_calculation_interval, - disk_usage_based_eviction, - test_remote_failures, - ondemand_download_behavior_treat_error_as_warn, - background_task_maximum_delay, - control_plane_api, - control_plane_api_token, - control_plane_emergency_mode, - heatmap_upload_concurrency, - secondary_download_concurrency, - ingest_batch_size, - max_vectored_read_bytes, - image_compression, - ephemeral_bytes_per_memory_kb, - l0_flush, - compact_level0_phase1_value_access, - virtual_file_direct_io, - io_buffer_alignment, - } - CUSTOM LOGIC - { - id: id, - // TenantConf is handled separately - default_tenant_conf: TenantConf::default(), - concurrent_tenant_warmup: ConfigurableSemaphore::new({ - self - .concurrent_tenant_warmup - .ok_or("concurrent_tenant_warmpup", - default.concurrent_tenant_warmup)? - }), - concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::new( - self - .concurrent_tenant_size_logical_size_queries - .ok_or("concurrent_tenant_size_logical_size_queries", - default.concurrent_tenant_size_logical_size_queries.clone())? - ), - eviction_task_immitated_concurrent_logical_size_queries: ConfigurableSemaphore::new( - // re-use `concurrent_tenant_size_logical_size_queries` - self - .concurrent_tenant_size_logical_size_queries - .ok_or("eviction_task_immitated_concurrent_logical_size_queries", - default.concurrent_tenant_size_logical_size_queries.clone())?, - ), - virtual_file_io_engine: match self.virtual_file_io_engine { - BuilderValue::Set(v) => v, - BuilderValue::NotSet => match crate::virtual_file::io_engine_feature_test().context("auto-detect virtual_file_io_engine")? { - io_engine::FeatureTestResult::PlatformPreferred(v) => v, // make no noise - io_engine::FeatureTestResult::Worse { engine, remark } => { - // TODO: bubble this up to the caller so we can tracing::warn! it. - eprintln!("auto-detected IO engine is not platform-preferred: engine={engine:?} remark={remark:?}"); - engine - } - }, - }, - } - )) - } -} - impl PageServerConf { // // Repository paths, relative to workdir. @@ -878,134 +300,135 @@ impl PageServerConf { /// /// This leaves any options not present in the file in the built-in defaults. pub fn parse_and_validate( - node_id: NodeId, - toml: &Document, + id: NodeId, + config_toml: pageserver_api::config::ConfigToml, workdir: &Utf8Path, ) -> anyhow::Result { - let mut builder = PageServerConfigBuilder::new(); - builder.workdir(workdir.to_owned()); + let pageserver_api::config::ConfigToml { + listen_pg_addr, + listen_http_addr, + availability_zone, + wait_lsn_timeout, + wal_redo_timeout, + superuser, + page_cache_size, + max_file_descriptors, + pg_distrib_dir, + http_auth_type, + pg_auth_type, + auth_validation_public_key_path, + remote_storage, + broker_endpoint, + broker_keepalive_interval, + log_format, + metric_collection_interval, + metric_collection_endpoint, + metric_collection_bucket, + synthetic_size_calculation_interval, + disk_usage_based_eviction, + test_remote_failures, + ondemand_download_behavior_treat_error_as_warn, + background_task_maximum_delay, + control_plane_api, + control_plane_api_token, + control_plane_emergency_mode, + heatmap_upload_concurrency, + secondary_download_concurrency, + ingest_batch_size, + max_vectored_read_bytes, + image_compression, + ephemeral_bytes_per_memory_kb, + compact_level0_phase1_value_access, + l0_flush, + virtual_file_direct_io, + concurrent_tenant_warmup, + concurrent_tenant_size_logical_size_queries, + virtual_file_io_engine, + io_buffer_alignment, + tenant_config, + } = config_toml; - let mut t_conf = TenantConfOpt::default(); + let mut conf = PageServerConf { + // ------------------------------------------------------------ + // fields that are already fully validated by the ConfigToml Deserialize impl + // ------------------------------------------------------------ + listen_pg_addr, + listen_http_addr, + availability_zone, + wait_lsn_timeout, + wal_redo_timeout, + superuser, + page_cache_size, + max_file_descriptors, + http_auth_type, + pg_auth_type, + auth_validation_public_key_path, + remote_storage_config: remote_storage, + broker_endpoint, + broker_keepalive_interval, + log_format, + metric_collection_interval, + metric_collection_endpoint, + metric_collection_bucket, + synthetic_size_calculation_interval, + disk_usage_based_eviction, + test_remote_failures, + ondemand_download_behavior_treat_error_as_warn, + background_task_maximum_delay, + control_plane_api, + control_plane_emergency_mode, + heatmap_upload_concurrency, + secondary_download_concurrency, + ingest_batch_size, + max_vectored_read_bytes, + image_compression, + ephemeral_bytes_per_memory_kb, + compact_level0_phase1_value_access, + virtual_file_direct_io, + io_buffer_alignment, - for (key, item) in toml.iter() { - match key { - "listen_pg_addr" => builder.listen_pg_addr(parse_toml_string(key, item)?), - "listen_http_addr" => builder.listen_http_addr(parse_toml_string(key, item)?), - "availability_zone" => builder.availability_zone(Some(parse_toml_string(key, item)?)), - "wait_lsn_timeout" => builder.wait_lsn_timeout(parse_toml_duration(key, item)?), - "wal_redo_timeout" => builder.wal_redo_timeout(parse_toml_duration(key, item)?), - "initial_superuser_name" => builder.superuser(parse_toml_string(key, item)?), - "page_cache_size" => builder.page_cache_size(parse_toml_u64(key, item)? as usize), - "max_file_descriptors" => { - builder.max_file_descriptors(parse_toml_u64(key, item)? as usize) - } - "pg_distrib_dir" => { - builder.pg_distrib_dir(Utf8PathBuf::from(parse_toml_string(key, item)?)) - } - "auth_validation_public_key_path" => builder.auth_validation_public_key_path(Some( - Utf8PathBuf::from(parse_toml_string(key, item)?), - )), - "http_auth_type" => builder.http_auth_type(parse_toml_from_str(key, item)?), - "pg_auth_type" => builder.pg_auth_type(parse_toml_from_str(key, item)?), - "remote_storage" => { - builder.remote_storage_config(Some(RemoteStorageConfig::from_toml(item).context("remote_storage")?)) - } - "tenant_config" => { - t_conf = TenantConfOpt::try_from(item.to_owned()).context(format!("failed to parse: '{key}'"))?; - } - "broker_endpoint" => builder.broker_endpoint(parse_toml_string(key, item)?.parse().context("failed to parse broker endpoint")?), - "broker_keepalive_interval" => builder.broker_keepalive_interval(parse_toml_duration(key, item)?), - "log_format" => builder.log_format( - LogFormat::from_config(&parse_toml_string(key, item)?)? - ), - "concurrent_tenant_warmup" => builder.concurrent_tenant_warmup({ - let input = parse_toml_string(key, item)?; - let permits = input.parse::().context("expected a number of initial permits, not {s:?}")?; - NonZeroUsize::new(permits).context("initial semaphore permits out of range: 0, use other configuration to disable a feature")? - }), - "concurrent_tenant_size_logical_size_queries" => builder.concurrent_tenant_size_logical_size_queries({ - let input = parse_toml_string(key, item)?; - let permits = input.parse::().context("expected a number of initial permits, not {s:?}")?; - NonZeroUsize::new(permits).context("initial semaphore permits out of range: 0, use other configuration to disable a feature")? - }), - "metric_collection_interval" => builder.metric_collection_interval(parse_toml_duration(key, item)?), - "metric_collection_endpoint" => { - let endpoint = parse_toml_string(key, item)?.parse().context("failed to parse metric_collection_endpoint")?; - builder.metric_collection_endpoint(Some(endpoint)); - }, - "metric_collection_bucket" => { - builder.metric_collection_bucket(Some(RemoteStorageConfig::from_toml(item)?)) - } - "synthetic_size_calculation_interval" => - builder.synthetic_size_calculation_interval(parse_toml_duration(key, item)?), - "test_remote_failures" => builder.test_remote_failures(parse_toml_u64(key, item)?), - "disk_usage_based_eviction" => { - tracing::info!("disk_usage_based_eviction: {:#?}", &item); - builder.disk_usage_based_eviction( - deserialize_from_item("disk_usage_based_eviction", item) - .context("parse disk_usage_based_eviction")? - ) - }, - "ondemand_download_behavior_treat_error_as_warn" => builder.ondemand_download_behavior_treat_error_as_warn(parse_toml_bool(key, item)?), - "background_task_maximum_delay" => builder.background_task_maximum_delay(parse_toml_duration(key, item)?), - "control_plane_api" => { - let parsed = parse_toml_string(key, item)?; - if parsed.is_empty() { - builder.control_plane_api(None) - } else { - builder.control_plane_api(Some(parsed.parse().context("failed to parse control plane URL")?)) + // ------------------------------------------------------------ + // fields that require additional validation or custom handling + // ------------------------------------------------------------ + workdir: workdir.to_owned(), + pg_distrib_dir: pg_distrib_dir.unwrap_or_else(|| { + std::env::current_dir() + .expect("current_dir() failed") + .try_into() + .expect("current_dir() is not a valid Utf8Path") + }), + control_plane_api_token: control_plane_api_token.map(SecretString::from), + id, + default_tenant_conf: tenant_config, + concurrent_tenant_warmup: ConfigurableSemaphore::new(concurrent_tenant_warmup), + concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::new( + concurrent_tenant_size_logical_size_queries, + ), + eviction_task_immitated_concurrent_logical_size_queries: ConfigurableSemaphore::new( + // re-use `concurrent_tenant_size_logical_size_queries` + concurrent_tenant_size_logical_size_queries, + ), + virtual_file_io_engine: match virtual_file_io_engine { + Some(v) => v, + None => match crate::virtual_file::io_engine_feature_test() + .context("auto-detect virtual_file_io_engine")? + { + io_engine::FeatureTestResult::PlatformPreferred(v) => v, // make no noise + io_engine::FeatureTestResult::Worse { engine, remark } => { + // TODO: bubble this up to the caller so we can tracing::warn! it. + eprintln!("auto-detected IO engine is not platform-preferred: engine={engine:?} remark={remark:?}"); + engine } }, - "control_plane_api_token" => { - let parsed = parse_toml_string(key, item)?; - if parsed.is_empty() { - builder.control_plane_api_token(None) - } else { - builder.control_plane_api_token(Some(parsed.into())) - } - }, - "control_plane_emergency_mode" => { - builder.control_plane_emergency_mode(parse_toml_bool(key, item)?) - }, - "heatmap_upload_concurrency" => { - builder.heatmap_upload_concurrency(parse_toml_u64(key, item)? as usize) - }, - "secondary_download_concurrency" => { - builder.secondary_download_concurrency(parse_toml_u64(key, item)? as usize) - }, - "ingest_batch_size" => builder.ingest_batch_size(parse_toml_u64(key, item)?), - "virtual_file_io_engine" => { - builder.virtual_file_io_engine(parse_toml_from_str("virtual_file_io_engine", item)?) - } - "max_vectored_read_bytes" => { - let bytes = parse_toml_u64("max_vectored_read_bytes", item)? as usize; - builder.get_max_vectored_read_bytes( - MaxVectoredReadBytes( - NonZeroUsize::new(bytes).expect("Max byte size of vectored read must be greater than 0"))) - } - "image_compression" => { - builder.get_image_compression(parse_toml_from_str("image_compression", item)?) - } - "ephemeral_bytes_per_memory_kb" => { - builder.get_ephemeral_bytes_per_memory_kb(parse_toml_u64("ephemeral_bytes_per_memory_kb", item)? as usize) - } - "l0_flush" => { - builder.l0_flush(utils::toml_edit_ext::deserialize_item(item).context("l0_flush")?) - } - "compact_level0_phase1_value_access" => { - builder.compact_level0_phase1_value_access(utils::toml_edit_ext::deserialize_item(item).context("compact_level0_phase1_value_access")?) - } - "virtual_file_direct_io" => { - builder.virtual_file_direct_io(utils::toml_edit_ext::deserialize_item(item).context("virtual_file_direct_io")?) - } - "io_buffer_alignment" => { - builder.io_buffer_alignment(parse_toml_u64("io_buffer_alignment", item)? as usize) - } - _ => bail!("unrecognized pageserver option '{key}'"), - } - } + }, + l0_flush: l0_flush + .map(crate::l0_flush::L0FlushConfig::from) + .unwrap_or_default(), + }; - let mut conf = builder.build(node_id).context("invalid config")?; + // ------------------------------------------------------------ + // custom validation code that covers more than one field in isolation + // ------------------------------------------------------------ if conf.http_auth_type == AuthType::NeonJWT || conf.pg_auth_type == AuthType::NeonJWT { let auth_validation_public_key_path = conf @@ -1019,10 +442,8 @@ impl PageServerConf { ); } - conf.default_tenant_conf = t_conf.merge(TenantConf::default()); - IndexEntry::validate_checkpoint_distance(conf.default_tenant_conf.checkpoint_distance) - .map_err(|msg| anyhow::anyhow!("{msg}")) + .map_err(anyhow::Error::msg) .with_context(|| { format!( "effective checkpoint distance is unsupported: {}", @@ -1042,130 +463,25 @@ impl PageServerConf { pub fn dummy_conf(repo_dir: Utf8PathBuf) -> Self { let pg_distrib_dir = Utf8PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../pg_install"); - PageServerConf { - id: NodeId(0), + let config_toml = pageserver_api::config::ConfigToml { wait_lsn_timeout: Duration::from_secs(60), wal_redo_timeout: Duration::from_secs(60), - page_cache_size: defaults::DEFAULT_PAGE_CACHE_SIZE, - max_file_descriptors: defaults::DEFAULT_MAX_FILE_DESCRIPTORS, - listen_pg_addr: defaults::DEFAULT_PG_LISTEN_ADDR.to_string(), - listen_http_addr: defaults::DEFAULT_HTTP_LISTEN_ADDR.to_string(), - availability_zone: None, - superuser: "cloud_admin".to_string(), - workdir: repo_dir, - pg_distrib_dir, - http_auth_type: AuthType::Trust, - pg_auth_type: AuthType::Trust, - auth_validation_public_key_path: None, - remote_storage_config: None, - default_tenant_conf: TenantConf::default(), - broker_endpoint: storage_broker::DEFAULT_ENDPOINT.parse().unwrap(), - broker_keepalive_interval: Duration::from_secs(5000), - log_format: LogFormat::from_str(defaults::DEFAULT_LOG_FORMAT).unwrap(), - concurrent_tenant_warmup: ConfigurableSemaphore::new( - NonZeroUsize::new(DEFAULT_CONCURRENT_TENANT_WARMUP) - .expect("Invalid default constant"), - ), - concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::default(), - eviction_task_immitated_concurrent_logical_size_queries: ConfigurableSemaphore::default( - ), + pg_distrib_dir: Some(pg_distrib_dir), metric_collection_interval: Duration::from_secs(60), - metric_collection_endpoint: defaults::DEFAULT_METRIC_COLLECTION_ENDPOINT, - metric_collection_bucket: None, synthetic_size_calculation_interval: Duration::from_secs(60), - disk_usage_based_eviction: None, - test_remote_failures: 0, - ondemand_download_behavior_treat_error_as_warn: false, background_task_maximum_delay: Duration::ZERO, - control_plane_api: None, - control_plane_api_token: None, - control_plane_emergency_mode: false, - heatmap_upload_concurrency: defaults::DEFAULT_HEATMAP_UPLOAD_CONCURRENCY, - secondary_download_concurrency: defaults::DEFAULT_SECONDARY_DOWNLOAD_CONCURRENCY, - ingest_batch_size: defaults::DEFAULT_INGEST_BATCH_SIZE, - virtual_file_io_engine: DEFAULT_VIRTUAL_FILE_IO_ENGINE.parse().unwrap(), - max_vectored_read_bytes: MaxVectoredReadBytes( - NonZeroUsize::new(defaults::DEFAULT_MAX_VECTORED_READ_BYTES) - .expect("Invalid default constant"), - ), - image_compression: defaults::DEFAULT_IMAGE_COMPRESSION.parse().unwrap(), - ephemeral_bytes_per_memory_kb: defaults::DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB, - l0_flush: L0FlushConfig::default(), - compact_level0_phase1_value_access: CompactL0Phase1ValueAccess::default(), - virtual_file_direct_io: virtual_file::DirectIoMode::default(), - io_buffer_alignment: defaults::DEFAULT_IO_BUFFER_ALIGNMENT, - } + ..Default::default() + }; + PageServerConf::parse_and_validate(NodeId(0), config_toml, &repo_dir).unwrap() } } -#[derive(Deserialize)] +#[derive(serde::Deserialize, serde::Serialize)] #[serde(deny_unknown_fields)] pub struct PageserverIdentity { pub id: NodeId, } -// Helper functions to parse a toml Item - -fn parse_toml_string(name: &str, item: &Item) -> Result { - let s = item - .as_str() - .with_context(|| format!("configure option {name} is not a string"))?; - Ok(s.to_string()) -} - -fn parse_toml_u64(name: &str, item: &Item) -> Result { - // A toml integer is signed, so it cannot represent the full range of an u64. That's OK - // for our use, though. - let i: i64 = item - .as_integer() - .with_context(|| format!("configure option {name} is not an integer"))?; - if i < 0 { - bail!("configure option {name} cannot be negative"); - } - Ok(i as u64) -} - -fn parse_toml_bool(name: &str, item: &Item) -> Result { - item.as_bool() - .with_context(|| format!("configure option {name} is not a bool")) -} - -fn parse_toml_duration(name: &str, item: &Item) -> Result { - let s = item - .as_str() - .with_context(|| format!("configure option {name} is not a string"))?; - - Ok(humantime::parse_duration(s)?) -} - -fn parse_toml_from_str(name: &str, item: &Item) -> anyhow::Result -where - T: FromStr, - ::Err: std::fmt::Display, -{ - let v = item - .as_str() - .with_context(|| format!("configure option {name} is not a string"))?; - T::from_str(v).map_err(|e| { - anyhow!( - "Failed to parse string as {parse_type} for configure option {name}: {e}", - parse_type = stringify!(T) - ) - }) -} - -fn deserialize_from_item(name: &str, item: &Item) -> anyhow::Result -where - T: serde::de::DeserializeOwned, -{ - // ValueDeserializer::new is not public, so use the ValueDeserializer's documented way - let deserializer = match item.clone().into_value() { - Ok(value) => value.into_deserializer(), - Err(item) => anyhow::bail!("toml_edit::Item '{item}' is not a toml_edit::Value"), - }; - T::deserialize(deserializer).with_context(|| format!("deserializing item for node {name}")) -} - /// Configurable semaphore permits setting. /// /// Does not allow semaphore permits to be zero, because at runtime initially zero permits and empty @@ -1227,469 +543,108 @@ impl ConfigurableSemaphore { #[cfg(test)] mod tests { - use std::{fs, num::NonZeroU32}; - use camino_tempfile::{tempdir, Utf8TempDir}; - use pageserver_api::models::EvictionPolicy; - use remote_storage::{RemoteStorageKind, S3Config}; - use utils::serde_percent::Percent; + use camino::Utf8PathBuf; + use utils::id::NodeId; - use super::*; - use crate::DEFAULT_PG_VERSION; - - const ALL_BASE_VALUES_TOML: &str = r#" -# Initial configuration file created by 'pageserver --init' - -listen_pg_addr = '127.0.0.1:64000' -listen_http_addr = '127.0.0.1:9898' - -wait_lsn_timeout = '111 s' -wal_redo_timeout = '111 s' - -page_cache_size = 444 -max_file_descriptors = 333 - -# initial superuser role name to use when creating a new tenant -initial_superuser_name = 'zzzz' - -metric_collection_interval = '222 s' -metric_collection_endpoint = 'http://localhost:80/metrics' -synthetic_size_calculation_interval = '333 s' - -log_format = 'json' -background_task_maximum_delay = '334 s' - -"#; + use super::PageServerConf; #[test] - fn parse_defaults() -> anyhow::Result<()> { - let tempdir = tempdir()?; - let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?; - let broker_endpoint = storage_broker::DEFAULT_ENDPOINT; - // we have to create dummy values to overcome the validation errors - let config_string = - format!("pg_distrib_dir='{pg_distrib_dir}'\nbroker_endpoint = '{broker_endpoint}'",); - let toml = config_string.parse()?; - - let parsed_config = PageServerConf::parse_and_validate(NodeId(10), &toml, &workdir) - .unwrap_or_else(|e| panic!("Failed to parse config '{config_string}', reason: {e:?}")); - - assert_eq!( - parsed_config, - PageServerConf { - id: NodeId(10), - listen_pg_addr: defaults::DEFAULT_PG_LISTEN_ADDR.to_string(), - listen_http_addr: defaults::DEFAULT_HTTP_LISTEN_ADDR.to_string(), - availability_zone: None, - wait_lsn_timeout: humantime::parse_duration(defaults::DEFAULT_WAIT_LSN_TIMEOUT)?, - wal_redo_timeout: humantime::parse_duration(defaults::DEFAULT_WAL_REDO_TIMEOUT)?, - superuser: defaults::DEFAULT_SUPERUSER.to_string(), - page_cache_size: defaults::DEFAULT_PAGE_CACHE_SIZE, - max_file_descriptors: defaults::DEFAULT_MAX_FILE_DESCRIPTORS, - workdir, - pg_distrib_dir, - http_auth_type: AuthType::Trust, - pg_auth_type: AuthType::Trust, - auth_validation_public_key_path: None, - remote_storage_config: None, - default_tenant_conf: TenantConf::default(), - broker_endpoint: storage_broker::DEFAULT_ENDPOINT.parse().unwrap(), - broker_keepalive_interval: humantime::parse_duration( - storage_broker::DEFAULT_KEEPALIVE_INTERVAL - )?, - log_format: LogFormat::from_str(defaults::DEFAULT_LOG_FORMAT).unwrap(), - concurrent_tenant_warmup: ConfigurableSemaphore::new( - NonZeroUsize::new(DEFAULT_CONCURRENT_TENANT_WARMUP).unwrap() - ), - concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::default(), - eviction_task_immitated_concurrent_logical_size_queries: - ConfigurableSemaphore::default(), - metric_collection_interval: humantime::parse_duration( - defaults::DEFAULT_METRIC_COLLECTION_INTERVAL - )?, - metric_collection_endpoint: defaults::DEFAULT_METRIC_COLLECTION_ENDPOINT, - metric_collection_bucket: None, - synthetic_size_calculation_interval: humantime::parse_duration( - defaults::DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL - )?, - disk_usage_based_eviction: None, - test_remote_failures: 0, - ondemand_download_behavior_treat_error_as_warn: false, - background_task_maximum_delay: humantime::parse_duration( - defaults::DEFAULT_BACKGROUND_TASK_MAXIMUM_DELAY - )?, - control_plane_api: None, - control_plane_api_token: None, - control_plane_emergency_mode: false, - heatmap_upload_concurrency: defaults::DEFAULT_HEATMAP_UPLOAD_CONCURRENCY, - secondary_download_concurrency: defaults::DEFAULT_SECONDARY_DOWNLOAD_CONCURRENCY, - ingest_batch_size: defaults::DEFAULT_INGEST_BATCH_SIZE, - virtual_file_io_engine: DEFAULT_VIRTUAL_FILE_IO_ENGINE.parse().unwrap(), - max_vectored_read_bytes: MaxVectoredReadBytes( - NonZeroUsize::new(defaults::DEFAULT_MAX_VECTORED_READ_BYTES) - .expect("Invalid default constant") - ), - image_compression: defaults::DEFAULT_IMAGE_COMPRESSION.parse().unwrap(), - ephemeral_bytes_per_memory_kb: defaults::DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB, - l0_flush: L0FlushConfig::default(), - compact_level0_phase1_value_access: CompactL0Phase1ValueAccess::default(), - virtual_file_direct_io: virtual_file::DirectIoMode::default(), - io_buffer_alignment: defaults::DEFAULT_IO_BUFFER_ALIGNMENT, - }, - "Correct defaults should be used when no config values are provided" - ); - - Ok(()) - } - - #[test] - fn parse_basic_config() -> anyhow::Result<()> { - let tempdir = tempdir()?; - let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?; - let broker_endpoint = storage_broker::DEFAULT_ENDPOINT; - - let config_string = format!( - "{ALL_BASE_VALUES_TOML}pg_distrib_dir='{pg_distrib_dir}'\nbroker_endpoint = '{broker_endpoint}'", - ); - let toml = config_string.parse()?; - - let parsed_config = PageServerConf::parse_and_validate(NodeId(10), &toml, &workdir) - .unwrap_or_else(|e| panic!("Failed to parse config '{config_string}', reason: {e:?}")); - - assert_eq!( - parsed_config, - PageServerConf { - id: NodeId(10), - listen_pg_addr: "127.0.0.1:64000".to_string(), - listen_http_addr: "127.0.0.1:9898".to_string(), - availability_zone: None, - wait_lsn_timeout: Duration::from_secs(111), - wal_redo_timeout: Duration::from_secs(111), - superuser: "zzzz".to_string(), - page_cache_size: 444, - max_file_descriptors: 333, - workdir, - pg_distrib_dir, - http_auth_type: AuthType::Trust, - pg_auth_type: AuthType::Trust, - auth_validation_public_key_path: None, - remote_storage_config: None, - default_tenant_conf: TenantConf::default(), - broker_endpoint: storage_broker::DEFAULT_ENDPOINT.parse().unwrap(), - broker_keepalive_interval: Duration::from_secs(5), - log_format: LogFormat::Json, - concurrent_tenant_warmup: ConfigurableSemaphore::new( - NonZeroUsize::new(DEFAULT_CONCURRENT_TENANT_WARMUP).unwrap() - ), - concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::default(), - eviction_task_immitated_concurrent_logical_size_queries: - ConfigurableSemaphore::default(), - metric_collection_interval: Duration::from_secs(222), - metric_collection_endpoint: Some(Url::parse("http://localhost:80/metrics")?), - metric_collection_bucket: None, - synthetic_size_calculation_interval: Duration::from_secs(333), - disk_usage_based_eviction: None, - test_remote_failures: 0, - ondemand_download_behavior_treat_error_as_warn: false, - background_task_maximum_delay: Duration::from_secs(334), - control_plane_api: None, - control_plane_api_token: None, - control_plane_emergency_mode: false, - heatmap_upload_concurrency: defaults::DEFAULT_HEATMAP_UPLOAD_CONCURRENCY, - secondary_download_concurrency: defaults::DEFAULT_SECONDARY_DOWNLOAD_CONCURRENCY, - ingest_batch_size: 100, - virtual_file_io_engine: DEFAULT_VIRTUAL_FILE_IO_ENGINE.parse().unwrap(), - max_vectored_read_bytes: MaxVectoredReadBytes( - NonZeroUsize::new(defaults::DEFAULT_MAX_VECTORED_READ_BYTES) - .expect("Invalid default constant") - ), - image_compression: defaults::DEFAULT_IMAGE_COMPRESSION.parse().unwrap(), - ephemeral_bytes_per_memory_kb: defaults::DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB, - l0_flush: L0FlushConfig::default(), - compact_level0_phase1_value_access: CompactL0Phase1ValueAccess::default(), - virtual_file_direct_io: virtual_file::DirectIoMode::default(), - io_buffer_alignment: defaults::DEFAULT_IO_BUFFER_ALIGNMENT, - }, - "Should be able to parse all basic config values correctly" - ); - - Ok(()) - } - - #[test] - fn parse_remote_fs_storage_config() -> anyhow::Result<()> { - let tempdir = tempdir()?; - let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?; - let broker_endpoint = "http://127.0.0.1:7777"; - - let local_storage_path = tempdir.path().join("local_remote_storage"); - - let identical_toml_declarations = &[ - format!( - r#"[remote_storage] -local_path = '{local_storage_path}'"#, - ), - format!("remote_storage={{local_path='{local_storage_path}'}}"), - ]; - - for remote_storage_config_str in identical_toml_declarations { - let config_string = format!( - r#"{ALL_BASE_VALUES_TOML} -pg_distrib_dir='{pg_distrib_dir}' -broker_endpoint = '{broker_endpoint}' - -{remote_storage_config_str}"#, - ); - - let toml = config_string.parse()?; - - let parsed_remote_storage_config = - PageServerConf::parse_and_validate(NodeId(10), &toml, &workdir) - .unwrap_or_else(|e| { - panic!("Failed to parse config '{config_string}', reason: {e:?}") - }) - .remote_storage_config - .expect("Should have remote storage config for the local FS"); - - assert_eq!( - parsed_remote_storage_config, - RemoteStorageConfig { - storage: RemoteStorageKind::LocalFs { local_path: local_storage_path.clone() }, - timeout: RemoteStorageConfig::DEFAULT_TIMEOUT, - }, - "Remote storage config should correctly parse the local FS config and fill other storage defaults" - ); - } - Ok(()) - } - - #[test] - fn parse_remote_s3_storage_config() -> anyhow::Result<()> { - let tempdir = tempdir()?; - let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?; - - let bucket_name = "some-sample-bucket".to_string(); - let bucket_region = "eu-north-1".to_string(); - let prefix_in_bucket = "test_prefix".to_string(); - let endpoint = "http://localhost:5000".to_string(); - let max_concurrent_syncs = NonZeroUsize::new(111).unwrap(); - let max_sync_errors = NonZeroU32::new(222).unwrap(); - let s3_concurrency_limit = NonZeroUsize::new(333).unwrap(); - let broker_endpoint = "http://127.0.0.1:7777"; - - let identical_toml_declarations = &[ - format!( - r#"[remote_storage] -max_concurrent_syncs = {max_concurrent_syncs} -max_sync_errors = {max_sync_errors} -bucket_name = '{bucket_name}' -bucket_region = '{bucket_region}' -prefix_in_bucket = '{prefix_in_bucket}' -endpoint = '{endpoint}' -concurrency_limit = {s3_concurrency_limit}"# - ), - format!( - "remote_storage={{max_concurrent_syncs={max_concurrent_syncs}, max_sync_errors={max_sync_errors}, bucket_name='{bucket_name}',\ - bucket_region='{bucket_region}', prefix_in_bucket='{prefix_in_bucket}', endpoint='{endpoint}', concurrency_limit={s3_concurrency_limit}}}", - ), - ]; - - for remote_storage_config_str in identical_toml_declarations { - let config_string = format!( - r#"{ALL_BASE_VALUES_TOML} -pg_distrib_dir='{pg_distrib_dir}' -broker_endpoint = '{broker_endpoint}' - -{remote_storage_config_str}"#, - ); - - let toml = config_string.parse()?; - - let parsed_remote_storage_config = - PageServerConf::parse_and_validate(NodeId(10), &toml, &workdir) - .unwrap_or_else(|e| { - panic!("Failed to parse config '{config_string}', reason: {e:?}") - }) - .remote_storage_config - .expect("Should have remote storage config for S3"); - - assert_eq!( - parsed_remote_storage_config, - RemoteStorageConfig { - storage: RemoteStorageKind::AwsS3(S3Config { - bucket_name: bucket_name.clone(), - bucket_region: bucket_region.clone(), - prefix_in_bucket: Some(prefix_in_bucket.clone()), - endpoint: Some(endpoint.clone()), - concurrency_limit: s3_concurrency_limit, - max_keys_per_list_response: None, - upload_storage_class: None, - }), - timeout: RemoteStorageConfig::DEFAULT_TIMEOUT, - }, - "Remote storage config should correctly parse the S3 config" - ); - } - Ok(()) - } - - #[test] - fn parse_incorrect_tenant_config() -> anyhow::Result<()> { - let config_string = r#" - [tenant_config] - checkpoint_distance = -1 # supposed to be an u64 - "# - .to_string(); - - let toml: Document = config_string.parse()?; - let item = toml.get("tenant_config").unwrap(); - let error = TenantConfOpt::try_from(item.to_owned()).unwrap_err(); - - let expected_error_str = "checkpoint_distance: invalid value: integer `-1`, expected u64"; - assert_eq!(error.to_string(), expected_error_str); - - Ok(()) - } - - #[test] - fn parse_override_tenant_config() -> anyhow::Result<()> { - let config_string = r#"tenant_config={ min_resident_size_override = 400 }"#.to_string(); - - let toml: Document = config_string.parse()?; - let item = toml.get("tenant_config").unwrap(); - let conf = TenantConfOpt::try_from(item.to_owned()).unwrap(); - - assert_eq!(conf.min_resident_size_override, Some(400)); - - Ok(()) - } - - #[test] - fn eviction_pageserver_config_parse() -> anyhow::Result<()> { - let tempdir = tempdir()?; - let (workdir, pg_distrib_dir) = prepare_fs(&tempdir)?; - - let pageserver_conf_toml = format!( - r#"pg_distrib_dir = "{pg_distrib_dir}" -metric_collection_endpoint = "http://sample.url" -metric_collection_interval = "10min" - -[disk_usage_based_eviction] -max_usage_pct = 80 -min_avail_bytes = 0 -period = "10s" - -[tenant_config] -evictions_low_residence_duration_metric_threshold = "20m" - -[tenant_config.eviction_policy] -kind = "LayerAccessThreshold" -period = "20m" -threshold = "20m" -"#, - ); - let toml: Document = pageserver_conf_toml.parse()?; - let conf = PageServerConf::parse_and_validate(NodeId(333), &toml, &workdir)?; - - assert_eq!(conf.pg_distrib_dir, pg_distrib_dir); - assert_eq!( - conf.metric_collection_endpoint, - Some("http://sample.url".parse().unwrap()) - ); - assert_eq!( - conf.metric_collection_interval, - Duration::from_secs(10 * 60) - ); - assert_eq!( - conf.default_tenant_conf - .evictions_low_residence_duration_metric_threshold, - Duration::from_secs(20 * 60) - ); - - // Assert that the node id provided by the indentity file (threaded - // through the call to [`PageServerConf::parse_and_validate`] is - // used. - assert_eq!(conf.id, NodeId(333)); - assert_eq!( - conf.disk_usage_based_eviction, - Some(DiskUsageEvictionTaskConfig { - max_usage_pct: Percent::new(80).unwrap(), - min_avail_bytes: 0, - period: Duration::from_secs(10), - #[cfg(feature = "testing")] - mock_statvfs: None, - eviction_order: Default::default(), - }) - ); - - match &conf.default_tenant_conf.eviction_policy { - EvictionPolicy::LayerAccessThreshold(eviction_threshold) => { - assert_eq!(eviction_threshold.period, Duration::from_secs(20 * 60)); - assert_eq!(eviction_threshold.threshold, Duration::from_secs(20 * 60)); - } - other => unreachable!("Unexpected eviction policy tenant settings: {other:?}"), - } - - Ok(()) - } - - #[test] - fn parse_imitation_only_pageserver_config() { - let tempdir = tempdir().unwrap(); - let (workdir, pg_distrib_dir) = prepare_fs(&tempdir).unwrap(); - - let pageserver_conf_toml = format!( - r#"pg_distrib_dir = "{pg_distrib_dir}" -metric_collection_endpoint = "http://sample.url" -metric_collection_interval = "10min" - -[tenant_config] -evictions_low_residence_duration_metric_threshold = "20m" - -[tenant_config.eviction_policy] -kind = "OnlyImitiate" -period = "20m" -threshold = "20m" -"#, - ); - let toml: Document = pageserver_conf_toml.parse().unwrap(); - let conf = PageServerConf::parse_and_validate(NodeId(222), &toml, &workdir).unwrap(); - - match &conf.default_tenant_conf.eviction_policy { - EvictionPolicy::OnlyImitiate(t) => { - assert_eq!(t.period, Duration::from_secs(20 * 60)); - assert_eq!(t.threshold, Duration::from_secs(20 * 60)); - } - other => unreachable!("Unexpected eviction policy tenant settings: {other:?}"), - } - } - - #[test] - fn empty_remote_storage_is_error() { - let tempdir = tempdir().unwrap(); - let (workdir, _) = prepare_fs(&tempdir).unwrap(); + fn test_empty_config_toml_is_valid() { + // we use Default impl of everything in this situation let input = r#" -remote_storage = {} "#; - let doc = toml_edit::Document::from_str(input).unwrap(); - let err = PageServerConf::parse_and_validate(NodeId(222), &doc, &workdir) - .expect_err("empty remote_storage field should fail, don't specify it if you want no remote_storage"); - assert!(format!("{err}").contains("remote_storage"), "{err}"); + let config_toml = toml_edit::de::from_str::(input) + .expect("empty config is valid"); + let workdir = Utf8PathBuf::from("/nonexistent"); + PageServerConf::parse_and_validate(NodeId(0), config_toml, &workdir) + .expect("parse_and_validate"); } - fn prepare_fs(tempdir: &Utf8TempDir) -> anyhow::Result<(Utf8PathBuf, Utf8PathBuf)> { - let tempdir_path = tempdir.path(); + /// If there's a typo in the pageserver config, we'd rather catch that typo + /// and fail pageserver startup than silently ignoring the typo, leaving whoever + /// made it in the believe that their config change is effective. + /// + /// The default in serde is to allow unknown fields, so, we rely + /// on developer+review discipline to add `deny_unknown_fields` when adding + /// new structs to the config, and these tests here as a regression test. + /// + /// The alternative to all of this would be to allow unknown fields in the config. + /// To catch them, we could have a config check tool or mgmt API endpoint that + /// compares the effective config with the TOML on disk and makes sure that + /// the on-disk TOML is a strict subset of the effective config. + mod unknown_fields_handling { + macro_rules! test { + ($short_name:ident, $input:expr) => { + #[test] + fn $short_name() { + let input = $input; + let err = toml_edit::de::from_str::(&input) + .expect_err("some_invalid_field is an invalid field"); + dbg!(&err); + assert!(err.to_string().contains("some_invalid_field")); + } + }; + } + use indoc::indoc; - let workdir = tempdir_path.join("workdir"); - fs::create_dir_all(&workdir)?; + test!( + toplevel, + indoc! {r#" + some_invalid_field = 23 + "#} + ); - let pg_distrib_dir = tempdir_path.join("pg_distrib"); - let pg_distrib_dir_versioned = pg_distrib_dir.join(format!("v{DEFAULT_PG_VERSION}")); - fs::create_dir_all(&pg_distrib_dir_versioned)?; - let postgres_bin_dir = pg_distrib_dir_versioned.join("bin"); - fs::create_dir_all(&postgres_bin_dir)?; - fs::write(postgres_bin_dir.join("postgres"), "I'm postgres, trust me")?; + test!( + toplevel_nested, + indoc! {r#" + [some_invalid_field] + foo = 23 + "#} + ); - Ok((workdir, pg_distrib_dir)) + test!( + disk_usage_based_eviction, + indoc! {r#" + [disk_usage_based_eviction] + some_invalid_field = 23 + "#} + ); + + test!( + tenant_config, + indoc! {r#" + [tenant_config] + some_invalid_field = 23 + "#} + ); + + test!( + l0_flush, + indoc! {r#" + [l0_flush] + mode = "direct" + some_invalid_field = 23 + "#} + ); + + // TODO: fix this => https://github.com/neondatabase/neon/issues/8915 + // test!( + // remote_storage_config, + // indoc! {r#" + // [remote_storage_config] + // local_path = "/nonexistent" + // some_invalid_field = 23 + // "#} + // ); + + test!( + compact_level0_phase1_value_access, + indoc! {r#" + [compact_level0_phase1_value_access] + mode = "streaming-kmerge" + some_invalid_field = 23 + "#} + ); } } diff --git a/pageserver/src/disk_usage_eviction_task.rs b/pageserver/src/disk_usage_eviction_task.rs index 5e4a49bc56..a58fa2c0b1 100644 --- a/pageserver/src/disk_usage_eviction_task.rs +++ b/pageserver/src/disk_usage_eviction_task.rs @@ -41,19 +41,15 @@ // - The `#[allow(dead_code)]` above various structs are to suppress warnings about only the Debug impl // reading these fields. We use the Debug impl for semi-structured logging, though. -use std::{ - sync::Arc, - time::{Duration, SystemTime}, -}; +use std::{sync::Arc, time::SystemTime}; use anyhow::Context; -use pageserver_api::shard::TenantShardId; +use pageserver_api::{config::DiskUsageEvictionTaskConfig, shard::TenantShardId}; use remote_storage::GenericRemoteStorage; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use tokio::time::Instant; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, instrument, warn, Instrument}; -use utils::serde_percent::Percent; use utils::{completion, id::TimelineId}; use crate::{ @@ -69,23 +65,9 @@ use crate::{ CancellableTask, DiskUsageEvictionTask, }; -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct DiskUsageEvictionTaskConfig { - pub max_usage_pct: Percent, - pub min_avail_bytes: u64, - #[serde(with = "humantime_serde")] - pub period: Duration, - #[cfg(feature = "testing")] - pub mock_statvfs: Option, - /// Select sorting for evicted layers - #[serde(default)] - pub eviction_order: EvictionOrder, -} - /// Selects the sort order for eviction candidates *after* per tenant `min_resident_size` /// partitioning. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "type", content = "args")] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum EvictionOrder { /// Order the layers to be evicted by how recently they have been accessed relatively within /// the set of resident layers of a tenant. @@ -96,23 +78,22 @@ pub enum EvictionOrder { /// we read tenants is deterministic. If we find the need to use this as `false`, we need /// to ensure nondeterminism by adding in a random number to break the /// `relative_last_activity==0.0` ties. - #[serde(default = "default_highest_layer_count_loses_first")] highest_layer_count_loses_first: bool, }, } -impl Default for EvictionOrder { - fn default() -> Self { - Self::RelativeAccessed { - highest_layer_count_loses_first: true, +impl From for EvictionOrder { + fn from(value: pageserver_api::config::EvictionOrder) -> Self { + match value { + pageserver_api::config::EvictionOrder::RelativeAccessed { + highest_layer_count_loses_first, + } => Self::RelativeAccessed { + highest_layer_count_loses_first, + }, } } } -fn default_highest_layer_count_loses_first() -> bool { - true -} - impl EvictionOrder { fn sort(&self, candidates: &mut [(EvictionPartition, EvictionCandidate)]) { use EvictionOrder::*; @@ -295,7 +276,7 @@ async fn disk_usage_eviction_task_iteration( storage, usage_pre, tenant_manager, - task_config.eviction_order, + task_config.eviction_order.into(), cancel, ) .await; @@ -1257,7 +1238,6 @@ mod filesystem_level_usage { #[test] fn max_usage_pct_pressure() { - use super::EvictionOrder; use super::Usage as _; use std::time::Duration; use utils::serde_percent::Percent; @@ -1269,7 +1249,7 @@ mod filesystem_level_usage { period: Duration::MAX, #[cfg(feature = "testing")] mock_statvfs: None, - eviction_order: EvictionOrder::default(), + eviction_order: pageserver_api::config::EvictionOrder::default(), }, total_bytes: 100_000, avail_bytes: 0, diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 90ae6c5557..d645f3b7b6 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -2076,7 +2076,7 @@ async fn disk_usage_eviction_run( evict_bytes: u64, #[serde(default)] - eviction_order: crate::disk_usage_eviction_task::EvictionOrder, + eviction_order: pageserver_api::config::EvictionOrder, } #[derive(Debug, Clone, Copy, serde::Serialize)] @@ -2112,7 +2112,7 @@ async fn disk_usage_eviction_run( &state.remote_storage, usage, &state.tenant_manager, - config.eviction_order, + config.eviction_order.into(), &cancel, ) .await; diff --git a/pageserver/src/l0_flush.rs b/pageserver/src/l0_flush.rs index 313a7961a6..491c9fb96c 100644 --- a/pageserver/src/l0_flush.rs +++ b/pageserver/src/l0_flush.rs @@ -1,9 +1,7 @@ use std::{num::NonZeroUsize, sync::Arc}; -#[derive(Debug, PartialEq, Eq, Clone, serde::Deserialize)] -#[serde(tag = "mode", rename_all = "kebab-case", deny_unknown_fields)] +#[derive(Debug, PartialEq, Eq, Clone)] pub enum L0FlushConfig { - #[serde(rename_all = "snake_case")] Direct { max_concurrency: NonZeroUsize }, } @@ -16,6 +14,16 @@ impl Default for L0FlushConfig { } } +impl From for L0FlushConfig { + fn from(config: pageserver_api::models::L0FlushConfig) -> Self { + match config { + pageserver_api::models::L0FlushConfig::Direct { max_concurrency } => { + Self::Direct { max_concurrency } + } + } + } +} + #[derive(Clone)] pub struct L0FlushGlobalState(Arc); diff --git a/pageserver/src/statvfs.rs b/pageserver/src/statvfs.rs index ede1791afa..5a6f6e5176 100644 --- a/pageserver/src/statvfs.rs +++ b/pageserver/src/statvfs.rs @@ -60,32 +60,7 @@ pub mod mock { use regex::Regex; use tracing::log::info; - #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] - #[serde(tag = "type")] - pub enum Behavior { - Success { - blocksize: u64, - total_blocks: u64, - name_filter: Option, - }, - Failure { - mocked_error: MockedError, - }, - } - - #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] - #[allow(clippy::upper_case_acronyms)] - pub enum MockedError { - EIO, - } - - impl From for nix::Error { - fn from(e: MockedError) -> Self { - match e { - MockedError::EIO => nix::Error::EIO, - } - } - } + pub use pageserver_api::config::statvfs::mock::Behavior; pub fn get(tenants_dir: &Utf8Path, behavior: &Behavior) -> nix::Result { info!("running mocked statvfs"); @@ -116,6 +91,7 @@ pub mod mock { block_size: *blocksize, }) } + #[cfg(feature = "testing")] Behavior::Failure { mocked_error } => Err((*mocked_error).into()), } } diff --git a/pageserver/src/tenant/config.rs b/pageserver/src/tenant/config.rs index 48ff17db94..7e0344666b 100644 --- a/pageserver/src/tenant/config.rs +++ b/pageserver/src/tenant/config.rs @@ -9,11 +9,10 @@ //! may lead to a data loss. //! use anyhow::bail; +pub(crate) use pageserver_api::config::TenantConfigToml as TenantConf; use pageserver_api::models::AuxFilePolicy; -use pageserver_api::models::CompactionAlgorithm; use pageserver_api::models::CompactionAlgorithmSettings; use pageserver_api::models::EvictionPolicy; -use pageserver_api::models::LsnLease; use pageserver_api::models::{self, ThrottleConfig}; use pageserver_api::shard::{ShardCount, ShardIdentity, ShardNumber, ShardStripeSize}; use serde::de::IntoDeserializer; @@ -23,50 +22,6 @@ use std::num::NonZeroU64; use std::time::Duration; use utils::generation::Generation; -pub mod defaults { - - // FIXME: This current value is very low. I would imagine something like 1 GB or 10 GB - // would be more appropriate. But a low value forces the code to be exercised more, - // which is good for now to trigger bugs. - // This parameter actually determines L0 layer file size. - pub const DEFAULT_CHECKPOINT_DISTANCE: u64 = 256 * 1024 * 1024; - pub const DEFAULT_CHECKPOINT_TIMEOUT: &str = "10 m"; - - // FIXME the below configs are only used by legacy algorithm. The new algorithm - // has different parameters. - - // Target file size, when creating image and delta layers. - // This parameter determines L1 layer file size. - pub const DEFAULT_COMPACTION_TARGET_SIZE: u64 = 128 * 1024 * 1024; - - pub const DEFAULT_COMPACTION_PERIOD: &str = "20 s"; - pub const DEFAULT_COMPACTION_THRESHOLD: usize = 10; - pub const DEFAULT_COMPACTION_ALGORITHM: super::CompactionAlgorithm = - super::CompactionAlgorithm::Legacy; - - pub const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024; - - // Large DEFAULT_GC_PERIOD is fine as long as PITR_INTERVAL is larger. - // If there's a need to decrease this value, first make sure that GC - // doesn't hold a layer map write lock for non-trivial operations. - // Relevant: https://github.com/neondatabase/neon/issues/3394 - pub const DEFAULT_GC_PERIOD: &str = "1 hr"; - pub const DEFAULT_IMAGE_CREATION_THRESHOLD: usize = 3; - pub const DEFAULT_PITR_INTERVAL: &str = "7 days"; - pub const DEFAULT_WALRECEIVER_CONNECT_TIMEOUT: &str = "10 seconds"; - pub const DEFAULT_WALRECEIVER_LAGGING_WAL_TIMEOUT: &str = "10 seconds"; - // The default limit on WAL lag should be set to avoid causing disconnects under high throughput - // scenarios: since the broker stats are updated ~1/s, a value of 1GiB should be sufficient for - // throughputs up to 1GiB/s per timeline. - pub const DEFAULT_MAX_WALRECEIVER_LSN_WAL_LAG: u64 = 1024 * 1024 * 1024; - pub const DEFAULT_EVICTIONS_LOW_RESIDENCE_DURATION_METRIC_THRESHOLD: &str = "24 hour"; - // By default ingest enough WAL for two new L0 layers before checking if new image - // image layers should be created. - pub const DEFAULT_IMAGE_LAYER_CREATION_CHECK_THRESHOLD: u8 = 2; - - pub const DEFAULT_INGEST_BATCH_SIZE: u64 = 100; -} - #[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)] pub(crate) enum AttachmentMode { /// Our generation is current as far as we know, and as far as we know we are the only attached @@ -281,96 +236,20 @@ impl LocationConf { } } -/// A tenant's calcuated configuration, which is the result of merging a -/// tenant's TenantConfOpt with the global TenantConf from PageServerConf. -/// -/// For storing and transmitting individual tenant's configuration, see -/// TenantConfOpt. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct TenantConf { - // Flush out an inmemory layer, if it's holding WAL older than this - // This puts a backstop on how much WAL needs to be re-digested if the - // page server crashes. - // This parameter actually determines L0 layer file size. - pub checkpoint_distance: u64, - // Inmemory layer is also flushed at least once in checkpoint_timeout to - // eventually upload WAL after activity is stopped. - #[serde(with = "humantime_serde")] - pub checkpoint_timeout: Duration, - // Target file size, when creating image and delta layers. - // This parameter determines L1 layer file size. - pub compaction_target_size: u64, - // How often to check if there's compaction work to be done. - // Duration::ZERO means automatic compaction is disabled. - #[serde(with = "humantime_serde")] - pub compaction_period: Duration, - // Level0 delta layer threshold for compaction. - pub compaction_threshold: usize, - pub compaction_algorithm: CompactionAlgorithmSettings, - // Determines how much history is retained, to allow - // branching and read replicas at an older point in time. - // The unit is #of bytes of WAL. - // Page versions older than this are garbage collected away. - pub gc_horizon: u64, - // Interval at which garbage collection is triggered. - // Duration::ZERO means automatic GC is disabled - #[serde(with = "humantime_serde")] - pub gc_period: Duration, - // Delta layer churn threshold to create L1 image layers. - pub image_creation_threshold: usize, - // Determines how much history is retained, to allow - // branching and read replicas at an older point in time. - // The unit is time. - // Page versions older than this are garbage collected away. - #[serde(with = "humantime_serde")] - pub pitr_interval: Duration, - /// Maximum amount of time to wait while opening a connection to receive wal, before erroring. - #[serde(with = "humantime_serde")] - pub walreceiver_connect_timeout: Duration, - /// Considers safekeepers stalled after no WAL updates were received longer than this threshold. - /// A stalled safekeeper will be changed to a newer one when it appears. - #[serde(with = "humantime_serde")] - pub lagging_wal_timeout: Duration, - /// Considers safekeepers lagging when their WAL is behind another safekeeper for more than this threshold. - /// A lagging safekeeper will be changed after `lagging_wal_timeout` time elapses since the last WAL update, - /// to avoid eager reconnects. - pub max_lsn_wal_lag: NonZeroU64, - pub eviction_policy: EvictionPolicy, - pub min_resident_size_override: Option, - // See the corresponding metric's help string. - #[serde(with = "humantime_serde")] - pub evictions_low_residence_duration_metric_threshold: Duration, - - /// If non-zero, the period between uploads of a heatmap from attached tenants. This - /// may be disabled if a Tenant will not have secondary locations: only secondary - /// locations will use the heatmap uploaded by attached locations. - #[serde(with = "humantime_serde")] - pub heatmap_period: Duration, - - /// If true then SLRU segments are dowloaded on demand, if false SLRU segments are included in basebackup - pub lazy_slru_download: bool, - - pub timeline_get_throttle: pageserver_api::models::ThrottleConfig, - - // How much WAL must be ingested before checking again whether a new image layer is required. - // Expresed in multiples of checkpoint distance. - pub image_layer_creation_check_threshold: u8, - - /// Switch to a new aux file policy. Switching this flag requires the user has not written any aux file into - /// the storage before, and this flag cannot be switched back. Otherwise there will be data corruptions. - /// There is a `last_aux_file_policy` flag which gets persisted in `index_part.json` once the first aux - /// file is written. - pub switch_aux_file_policy: AuxFilePolicy, - - /// The length for an explicit LSN lease request. - /// Layers needed to reconstruct pages at LSN will not be GC-ed during this interval. - #[serde(with = "humantime_serde")] - pub lsn_lease_length: Duration, - - /// The length for an implicit LSN lease granted as part of `get_lsn_by_timestamp` request. - /// Layers needed to reconstruct pages at LSN will not be GC-ed during this interval. - #[serde(with = "humantime_serde")] - pub lsn_lease_length_for_ts: Duration, +impl Default for LocationConf { + // TODO: this should be removed once tenant loading can guarantee that we are never + // loading from a directory without a configuration. + // => tech debt since https://github.com/neondatabase/neon/issues/1555 + fn default() -> Self { + Self { + mode: LocationMode::Attached(AttachedLocationConfig { + generation: Generation::none(), + attach_mode: AttachmentMode::Single, + }), + tenant_conf: TenantConfOpt::default(), + shard: ShardIdentity::unsharded(), + } + } } /// Same as TenantConf, but this struct preserves the information about @@ -545,51 +424,6 @@ impl TenantConfOpt { } } -impl Default for TenantConf { - fn default() -> Self { - use defaults::*; - Self { - checkpoint_distance: DEFAULT_CHECKPOINT_DISTANCE, - checkpoint_timeout: humantime::parse_duration(DEFAULT_CHECKPOINT_TIMEOUT) - .expect("cannot parse default checkpoint timeout"), - compaction_target_size: DEFAULT_COMPACTION_TARGET_SIZE, - compaction_period: humantime::parse_duration(DEFAULT_COMPACTION_PERIOD) - .expect("cannot parse default compaction period"), - compaction_threshold: DEFAULT_COMPACTION_THRESHOLD, - compaction_algorithm: CompactionAlgorithmSettings { - kind: DEFAULT_COMPACTION_ALGORITHM, - }, - gc_horizon: DEFAULT_GC_HORIZON, - gc_period: humantime::parse_duration(DEFAULT_GC_PERIOD) - .expect("cannot parse default gc period"), - image_creation_threshold: DEFAULT_IMAGE_CREATION_THRESHOLD, - pitr_interval: humantime::parse_duration(DEFAULT_PITR_INTERVAL) - .expect("cannot parse default PITR interval"), - walreceiver_connect_timeout: humantime::parse_duration( - DEFAULT_WALRECEIVER_CONNECT_TIMEOUT, - ) - .expect("cannot parse default walreceiver connect timeout"), - lagging_wal_timeout: humantime::parse_duration(DEFAULT_WALRECEIVER_LAGGING_WAL_TIMEOUT) - .expect("cannot parse default walreceiver lagging wal timeout"), - max_lsn_wal_lag: NonZeroU64::new(DEFAULT_MAX_WALRECEIVER_LSN_WAL_LAG) - .expect("cannot parse default max walreceiver Lsn wal lag"), - eviction_policy: EvictionPolicy::NoEviction, - min_resident_size_override: None, - evictions_low_residence_duration_metric_threshold: humantime::parse_duration( - DEFAULT_EVICTIONS_LOW_RESIDENCE_DURATION_METRIC_THRESHOLD, - ) - .expect("cannot parse default evictions_low_residence_duration_metric_threshold"), - heatmap_period: Duration::ZERO, - lazy_slru_download: false, - timeline_get_throttle: crate::tenant::throttle::Config::disabled(), - image_layer_creation_check_threshold: DEFAULT_IMAGE_LAYER_CREATION_CHECK_THRESHOLD, - switch_aux_file_policy: AuxFilePolicy::default_tenant_config(), - lsn_lease_length: LsnLease::DEFAULT_LENGTH, - lsn_lease_length_for_ts: LsnLease::DEFAULT_LENGTH_FOR_TS, - } - } -} - impl TryFrom<&'_ models::TenantConfig> for TenantConfOpt { type Error = anyhow::Error; diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index b8e9a98149..6a2cd94232 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -39,7 +39,7 @@ use crate::tenant::disk_btree::{ use crate::tenant::storage_layer::layer::S3_UPLOAD_LIMIT; use crate::tenant::timeline::GetVectoredError; use crate::tenant::vectored_blob_io::{ - BlobFlag, MaxVectoredReadBytes, StreamingVectoredReadPlanner, VectoredBlobReader, VectoredRead, + BlobFlag, StreamingVectoredReadPlanner, VectoredBlobReader, VectoredRead, VectoredReadCoalesceMode, VectoredReadPlanner, }; use crate::tenant::PageReconstructError; @@ -52,6 +52,7 @@ use bytes::BytesMut; use camino::{Utf8Path, Utf8PathBuf}; use futures::StreamExt; use itertools::Itertools; +use pageserver_api::config::MaxVectoredReadBytes; use pageserver_api::keyspace::KeySpace; use pageserver_api::models::ImageCompressionAlgorithm; use pageserver_api::shard::TenantShardId; diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index 4a095c564d..77ce1ae670 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -34,8 +34,7 @@ use crate::tenant::disk_btree::{ }; use crate::tenant::timeline::GetVectoredError; use crate::tenant::vectored_blob_io::{ - BlobFlag, MaxVectoredReadBytes, StreamingVectoredReadPlanner, VectoredBlobReader, VectoredRead, - VectoredReadPlanner, + BlobFlag, StreamingVectoredReadPlanner, VectoredBlobReader, VectoredRead, VectoredReadPlanner, }; use crate::tenant::{PageReconstructError, Timeline}; use crate::virtual_file::owned_buffers_io::io_buf_ext::IoBufExt; @@ -46,6 +45,7 @@ use bytes::{Bytes, BytesMut}; use camino::{Utf8Path, Utf8PathBuf}; use hex; use itertools::Itertools; +use pageserver_api::config::MaxVectoredReadBytes; use pageserver_api::keyspace::KeySpace; use pageserver_api::shard::{ShardIdentity, TenantShardId}; use rand::{distributions::Alphanumeric, Rng}; diff --git a/pageserver/src/tenant/storage_layer/inmemory_layer.rs b/pageserver/src/tenant/storage_layer/inmemory_layer.rs index 2c19e5b19f..e487bee1f2 100644 --- a/pageserver/src/tenant/storage_layer/inmemory_layer.rs +++ b/pageserver/src/tenant/storage_layer/inmemory_layer.rs @@ -215,7 +215,7 @@ impl IndexEntry { const _ASSERT_DEFAULT_CHECKPOINT_DISTANCE_IS_VALID: () = { let res = Self::validate_checkpoint_distance( - crate::tenant::config::defaults::DEFAULT_CHECKPOINT_DISTANCE, + pageserver_api::config::tenant_conf_defaults::DEFAULT_CHECKPOINT_DISTANCE, ); if res.is_err() { panic!("default checkpoint distance is valid") diff --git a/pageserver/src/tenant/tasks.rs b/pageserver/src/tenant/tasks.rs index f5680ced90..478e9bb4f0 100644 --- a/pageserver/src/tenant/tasks.rs +++ b/pageserver/src/tenant/tasks.rs @@ -10,7 +10,6 @@ use crate::context::{DownloadBehavior, RequestContext}; use crate::metrics::TENANT_TASK_EVENTS; use crate::task_mgr; use crate::task_mgr::{TaskKind, BACKGROUND_RUNTIME}; -use crate::tenant::config::defaults::DEFAULT_COMPACTION_PERIOD; use crate::tenant::throttle::Stats; use crate::tenant::timeline::CompactionError; use crate::tenant::{Tenant, TenantState}; @@ -456,9 +455,11 @@ async fn ingest_housekeeping_loop(tenant: Arc, cancel: CancellationToken // If compaction period is set to zero (to disable it), then we will use a reasonable default let period = if period == Duration::ZERO { - humantime::Duration::from_str(DEFAULT_COMPACTION_PERIOD) - .unwrap() - .into() + humantime::Duration::from_str( + pageserver_api::config::tenant_conf_defaults::DEFAULT_COMPACTION_PERIOD, + ) + .unwrap() + .into() } else { period }; diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 3b8f19a6c0..262dccac7d 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -66,7 +66,6 @@ use std::{ use crate::{ aux_file::AuxFileSizeEstimator, tenant::{ - config::defaults::DEFAULT_PITR_INTERVAL, layer_map::{LayerMap, SearchResult}, metadata::TimelineMetadata, storage_layer::{inmemory_layer::IndexEntry, PersistentLayerDesc}, @@ -102,6 +101,7 @@ use crate::{ pgdatadir_mapping::{AuxFilesDirectory, DirectoryKind}, virtual_file::{MaybeFatalIo, VirtualFile}, }; +use pageserver_api::config::tenant_conf_defaults::DEFAULT_PITR_INTERVAL; use crate::config::PageServerConf; use crate::keyspace::{KeyPartitioning, KeySpace}; diff --git a/pageserver/src/tenant/timeline/compaction.rs b/pageserver/src/tenant/timeline/compaction.rs index aad75ac59c..6b9c8386f7 100644 --- a/pageserver/src/tenant/timeline/compaction.rs +++ b/pageserver/src/tenant/timeline/compaction.rs @@ -19,6 +19,7 @@ use bytes::Bytes; use enumset::EnumSet; use fail::fail_point; use itertools::Itertools; +use pageserver_api::config::{CompactL0BypassPageCacheValidation, CompactL0Phase1ValueAccess}; use pageserver_api::key::KEY_SIZE; use pageserver_api::keyspace::ShardedRange; use pageserver_api::shard::{ShardCount, ShardIdentity, TenantShardId}; @@ -29,7 +30,6 @@ use utils::id::TimelineId; use crate::context::{AccessStatsBehavior, RequestContext, RequestContextBuilder}; use crate::page_cache; -use crate::tenant::config::defaults::{DEFAULT_CHECKPOINT_DISTANCE, DEFAULT_COMPACTION_THRESHOLD}; use crate::tenant::remote_timeline_client::WaitCompletionError; use crate::tenant::storage_layer::merge_iterator::MergeIterator; use crate::tenant::storage_layer::split_writer::{ @@ -43,6 +43,9 @@ use crate::tenant::timeline::{drop_rlock, DeltaLayerWriter, ImageLayerWriter}; use crate::tenant::timeline::{Layer, ResidentLayer}; use crate::tenant::DeltaLayer; use crate::virtual_file::{MaybeFatalIo, VirtualFile}; +use pageserver_api::config::tenant_conf_defaults::{ + DEFAULT_CHECKPOINT_DISTANCE, DEFAULT_COMPACTION_THRESHOLD, +}; use crate::keyspace::KeySpace; use crate::repository::{Key, Value}; @@ -1433,43 +1436,6 @@ impl TryFrom for CompactLevel0Phase1Stats { } } -#[derive(Debug, PartialEq, Eq, Clone, serde::Deserialize, serde::Serialize)] -#[serde(tag = "mode", rename_all = "kebab-case", deny_unknown_fields)] -pub enum CompactL0Phase1ValueAccess { - /// The old way. - PageCachedBlobIo, - /// The new way. - StreamingKmerge { - /// If set, we run both the old way and the new way, validate that - /// they are identical (=> [`CompactL0BypassPageCacheValidation`]), - /// and if the validation fails, - /// - in tests: fail them with a panic or - /// - in prod, log a rate-limited warning and use the old way's results. - /// - /// If not set, we only run the new way and trust its results. - validate: Option, - }, -} - -/// See [`CompactL0Phase1ValueAccess::StreamingKmerge`]. -#[derive(Debug, PartialEq, Eq, Clone, serde::Deserialize, serde::Serialize)] -#[serde(rename_all = "kebab-case")] -pub enum CompactL0BypassPageCacheValidation { - /// Validate that the series of (key, lsn) pairs are the same. - KeyLsn, - /// Validate that the entire output of old and new way is identical. - KeyLsnValue, -} - -impl Default for CompactL0Phase1ValueAccess { - fn default() -> Self { - CompactL0Phase1ValueAccess::StreamingKmerge { - // TODO(https://github.com/neondatabase/neon/issues/8184): change to None once confident - validate: Some(CompactL0BypassPageCacheValidation::KeyLsnValue), - } - } -} - impl Timeline { /// Entry point for new tiered compaction algorithm. /// diff --git a/pageserver/src/tenant/vectored_blob_io.rs b/pageserver/src/tenant/vectored_blob_io.rs index 146bcf0e35..4d51dc442d 100644 --- a/pageserver/src/tenant/vectored_blob_io.rs +++ b/pageserver/src/tenant/vectored_blob_io.rs @@ -16,7 +16,6 @@ //! Note that the vectored blob api does *not* go through the page cache. use std::collections::BTreeMap; -use std::num::NonZeroUsize; use bytes::BytesMut; use pageserver_api::key::Key; @@ -29,9 +28,6 @@ use crate::context::RequestContext; use crate::tenant::blob_io::{BYTE_UNCOMPRESSED, BYTE_ZSTD, LEN_COMPRESSION_BIT_MASK}; use crate::virtual_file::{self, VirtualFile}; -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub struct MaxVectoredReadBytes(pub NonZeroUsize); - /// Metadata bundled with the start and end offset of a blob. #[derive(Copy, Clone, Debug)] pub struct BlobMeta { diff --git a/pageserver/src/virtual_file.rs b/pageserver/src/virtual_file.rs index 97d966e2da..ed6ff86c10 100644 --- a/pageserver/src/virtual_file.rs +++ b/pageserver/src/virtual_file.rs @@ -10,7 +10,6 @@ //! This is similar to PostgreSQL's virtual file descriptor facility in //! src/backend/storage/file/fd.c //! -use crate::config::defaults::DEFAULT_IO_BUFFER_ALIGNMENT; use crate::context::RequestContext; use crate::metrics::{StorageIoOperation, STORAGE_IO_SIZE, STORAGE_IO_TIME_METRIC}; @@ -19,6 +18,7 @@ use crate::tenant::TENANTS_SEGMENT_NAME; use camino::{Utf8Path, Utf8PathBuf}; use once_cell::sync::OnceCell; use owned_buffers_io::io_buf_ext::FullSlice; +use pageserver_api::config::defaults::DEFAULT_IO_BUFFER_ALIGNMENT; use pageserver_api::shard::TenantShardId; use std::fs::File; use std::io::{Error, ErrorKind, Seek, SeekFrom}; diff --git a/pageserver/src/virtual_file/io_engine.rs b/pageserver/src/virtual_file/io_engine.rs index faef1ba9ff..ccde90ee1a 100644 --- a/pageserver/src/virtual_file/io_engine.rs +++ b/pageserver/src/virtual_file/io_engine.rs @@ -84,9 +84,14 @@ pub(crate) fn get() -> IoEngine { } }, Err(std::env::VarError::NotPresent) => { - crate::config::defaults::DEFAULT_VIRTUAL_FILE_IO_ENGINE - .parse() - .unwrap() + #[cfg(target_os = "linux")] + { + IoEngineKind::TokioEpollUring + } + #[cfg(not(target_os = "linux"))] + { + IoEngineKind::StdFs + } } Err(std::env::VarError::NotUnicode(_)) => { panic!("env var {env_var_name} is not unicode"); diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 890538b86a..2df45a7e0e 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -24,7 +24,20 @@ from functools import cached_property, partial from itertools import chain, product from pathlib import Path from types import TracebackType -from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Tuple, Type, Union, cast +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + Optional, + Tuple, + Type, + TypeVar, + Union, + cast, +) from urllib.parse import quote, urlparse import asyncpg @@ -90,6 +103,8 @@ from fixtures.utils import AuxFileStore as AuxFileStore # reexport from .neon_api import NeonAPI, NeonApiEndpoint +T = TypeVar("T") + """ This file contains pytest fixtures. A fixture is a test resource that can be summoned by placing its name in the test's arguments. @@ -2986,16 +3001,17 @@ class NeonPageserver(PgProtocol, LogUtils): def config_toml_path(self) -> Path: return self.workdir / "pageserver.toml" - def edit_config_toml(self, edit_fn: Callable[[Dict[str, Any]], None]): + def edit_config_toml(self, edit_fn: Callable[[Dict[str, Any]], T]) -> T: """ Edit the pageserver's config toml file in place. """ path = self.config_toml_path with open(path, "r") as f: config = toml.load(f) - edit_fn(config) + res = edit_fn(config) with open(path, "w") as f: toml.dump(config, f) + return res def patch_config_toml_nonrecursive(self, patch: Dict[str, Any]) -> Dict[str, Any]: """ diff --git a/test_runner/regress/test_pageserver_generations.py b/test_runner/regress/test_pageserver_generations.py index 73af7950f1..ebf58d2bd1 100644 --- a/test_runner/regress/test_pageserver_generations.py +++ b/test_runner/regress/test_pageserver_generations.py @@ -142,11 +142,10 @@ def test_generations_upgrade(neon_env_builder: NeonEnvBuilder): # We will start a pageserver with no control_plane_api set, so it won't be able to self-register env.storage_controller.node_register(env.pageserver) - replaced_config = env.pageserver.patch_config_toml_nonrecursive( - { - "control_plane_api": "", - } - ) + def remove_control_plane_api_field(config): + return config.pop("control_plane_api") + + control_plane_api = env.pageserver.edit_config_toml(remove_control_plane_api_field) env.pageserver.start() env.storage_controller.node_configure(env.pageserver.id, {"availability": "Active"}) @@ -179,7 +178,11 @@ def test_generations_upgrade(neon_env_builder: NeonEnvBuilder): env.pageserver.stop() # Starting without the override that disabled control_plane_api - env.pageserver.patch_config_toml_nonrecursive(replaced_config) + env.pageserver.patch_config_toml_nonrecursive( + { + "control_plane_api": control_plane_api, + } + ) env.pageserver.start() generate_uploads_and_deletions(env, pageserver=env.pageserver, init=False) diff --git a/test_runner/regress/test_timeline_size.py b/test_runner/regress/test_timeline_size.py index 642b9e449b..9bf5f8680b 100644 --- a/test_runner/regress/test_timeline_size.py +++ b/test_runner/regress/test_timeline_size.py @@ -733,7 +733,7 @@ def test_ondemand_activation(neon_env_builder: NeonEnvBuilder): # We will run with the limit set to 1, so that once we have one tenant stuck # in a pausable failpoint, the rest are prevented from proceeding through warmup. - neon_env_builder.pageserver_config_override = "concurrent_tenant_warmup = '1'" + neon_env_builder.pageserver_config_override = "concurrent_tenant_warmup = 1" env = neon_env_builder.init_start() pageserver_http = env.pageserver.http_client() @@ -984,7 +984,7 @@ def test_timeline_logical_size_task_priority(neon_env_builder: NeonEnvBuilder): def test_eager_attach_does_not_queue_up(neon_env_builder: NeonEnvBuilder): - neon_env_builder.pageserver_config_override = "concurrent_tenant_warmup = '1'" + neon_env_builder.pageserver_config_override = "concurrent_tenant_warmup = 1" env = neon_env_builder.init_start() @@ -1062,7 +1062,7 @@ def test_eager_attach_does_not_queue_up(neon_env_builder: NeonEnvBuilder): @pytest.mark.parametrize("activation_method", ["endpoint", "branch", "delete"]) def test_lazy_attach_activation(neon_env_builder: NeonEnvBuilder, activation_method: str): # env.initial_tenant will take up this permit when attaching with lazy because of a failpoint activated after restart - neon_env_builder.pageserver_config_override = "concurrent_tenant_warmup = '1'" + neon_env_builder.pageserver_config_override = "concurrent_tenant_warmup = 1" env = neon_env_builder.init_start() From efe03d5a1ccce8e0f53e733d61fd0e3d0dd904f8 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 5 Sep 2024 16:29:48 +0300 Subject: [PATCH 26/45] build: sync between benchies (#8919) Sometimes, the benchmarks fail to start up pageserver in 10s without any obvious reason. Benchmarks run sequentially on otherwise idle runners. Try running `sync(2)` after each bench to force a cleaner slate. Implement this via: - SYNC_AFTER_EACH_TEST environment variable enabled autouse fixture - autouse fixture seems to be outermost fixture, so it works as expected - set SYNC_AFTER_EACH_TEST=true for benchmarks in build_and_test workflow Evidence: https://neon-github-public-dev.s3.amazonaws.com/reports/main/10678984691/index.html#suites/5008d72a1ba3c0d618a030a938fc035c/1210266507534c0f/ --------- Co-authored-by: Alexander Bayandin --- .github/workflows/build_and_test.yml | 1 + test_runner/fixtures/compare_fixtures.py | 26 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 53d33b420f..ee5fd1b0c6 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -286,6 +286,7 @@ jobs: PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" TEST_RESULT_CONNSTR: "${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }}" PAGESERVER_VIRTUAL_FILE_IO_ENGINE: tokio-epoll-uring + SYNC_AFTER_EACH_TEST: true # XXX: no coverage data handling here, since benchmarks are run on release builds, # while coverage is currently collected for the debug ones diff --git a/test_runner/fixtures/compare_fixtures.py b/test_runner/fixtures/compare_fixtures.py index 98a9dd7184..7c4a8db36f 100644 --- a/test_runner/fixtures/compare_fixtures.py +++ b/test_runner/fixtures/compare_fixtures.py @@ -1,3 +1,5 @@ +import os +import time from abc import ABC, abstractmethod from contextlib import _GeneratorContextManager, contextmanager @@ -8,6 +10,7 @@ import pytest from _pytest.fixtures import FixtureRequest from fixtures.benchmark_fixture import MetricReport, NeonBenchmarker +from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnv, PgBin, @@ -333,3 +336,26 @@ def neon_with_baseline(request: FixtureRequest) -> PgCompare: fixture = request.getfixturevalue(request.param) assert isinstance(fixture, PgCompare), f"test error: fixture {fixture} is not PgCompare" return fixture + + +@pytest.fixture(scope="function", autouse=True) +def sync_after_each_test(): + # The fixture calls `sync(2)` after each test if `SYNC_AFTER_EACH_TEST` env var is `true` + # + # In CI, `SYNC_AFTER_EACH_TEST` is set to `true` only for benchmarks (`test_runner/performance`) + # that are run on self-hosted runners because some of these tests are pretty write-heavy + # and create issues to start the processes within 10s + key = "SYNC_AFTER_EACH_TEST" + enabled = os.environ.get(key) == "true" + + yield + + if not enabled: + # regress test, or running locally + return + + start = time.time() + # we only run benches on unices, the method might not exist on windows + os.sync() + elapsed = time.time() - start + log.info(f"called sync after test {elapsed=}") From ebddda5b7f85587998df00dbf7dc88679459b494 Mon Sep 17 00:00:00 2001 From: vladov Date: Thu, 5 Sep 2024 08:06:57 -0700 Subject: [PATCH 27/45] Fix precedence issue causing yielding loop to never yield. (#8922) There is a bug in `yielding_loop` that causes it to never yield. ## Summary of changes Fixed the bug. `i + 1 % interval == 0` will always evaluate to `i + 1 == 0` which is false ([Playground](https://play.rust-lang.org/?version=stable&mode=debug&edition=2021&gist=68e6ca393a02113cb7720115c2842e75)). This function is called in 2 places [here](https://github.com/neondatabase/neon/blob/99fa1c36004d710c65a47ffefaf66b4b5c6b4ce1/pageserver/src/tenant/secondary/scheduler.rs#L389) and [here](https://github.com/neondatabase/neon/blob/99fa1c36004d710c65a47ffefaf66b4b5c6b4ce1/pageserver/src/tenant/secondary/heatmap_uploader.rs#L152) with `interval == 1000` in both cases. This may change the performance of the system since now we are yielding to tokio. Also, this may expose undefined behavior since it is now possible for tasks to be moved between threads/whatever tokio does to tasks. However, this was the intention of the author of the code. --- libs/utils/src/yielding_loop.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/utils/src/yielding_loop.rs b/libs/utils/src/yielding_loop.rs index 963279eb4c..41c4cee45d 100644 --- a/libs/utils/src/yielding_loop.rs +++ b/libs/utils/src/yielding_loop.rs @@ -23,7 +23,7 @@ where for (i, item) in iter.enumerate() { visitor(item); - if i + 1 % interval == 0 { + if (i + 1) % interval == 0 { tokio::task::yield_now().await; if cancel.is_cancelled() { return Err(YieldingLoopError::Cancelled); From fd12dd942f61a0a22016fa219f4b3a87c81dc0b0 Mon Sep 17 00:00:00 2001 From: Stefan Radig Date: Thu, 5 Sep 2024 17:48:51 +0200 Subject: [PATCH 28/45] Add installation instructions for m4 on mac (#8929) ## Problem Building on MacOS failed due to missing m4. Although a window was popping up claiming to install m4, this was not helping. ## Summary of changes Add instructions to install m4 using brew and link it (thanks to Folke for helping). --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 735edef0fc..b54956f773 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,12 @@ brew install protobuf openssl flex bison icu4c pkg-config echo 'export PATH="$(brew --prefix openssl)/bin:$PATH"' >> ~/.zshrc ``` +If you get errors about missing `m4` you may have to install it manually: +``` +brew install m4 +brew link --force m4 +``` + 2. [Install Rust](https://www.rust-lang.org/tools/install) ``` # recommended approach from https://www.rust-lang.org/tools/install From 04f99a87bfee4da41df2bd5724e73b3646c2bf3e Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Thu, 5 Sep 2024 19:14:21 +0100 Subject: [PATCH 29/45] storcon: make pageserver AZ id mandatory (#8856) ## Problem https://github.com/neondatabase/neon/pull/8852 introduced a new nullable column for the `nodes` table: `availability_zone_id` ## Summary of changes * Make neon local and the test suite always provide an az id * Make the az id field in the ps registration request mandatory * Migrate the column to non-nullable and adjust in memory state accordingly * Remove the code that was used to populate the az id for pre-existing nodes --- Dockerfile | 1 + control_plane/storcon_cli/src/main.rs | 2 +- libs/pageserver_api/src/controller_api.rs | 2 +- pageserver/src/control_plane_client.rs | 24 +++++++--- .../down.sql | 1 + .../up.sql | 1 + storage_controller/src/node.rs | 23 +++------- storage_controller/src/persistence.rs | 28 +----------- storage_controller/src/scheduler.rs | 2 +- storage_controller/src/schema.rs | 2 +- storage_controller/src/service.rs | 44 ++----------------- test_runner/fixtures/neon_fixtures.py | 3 ++ 12 files changed, 41 insertions(+), 92 deletions(-) create mode 100644 storage_controller/migrations/2024-08-28-150530_pageserver_az_not_null/down.sql create mode 100644 storage_controller/migrations/2024-08-28-150530_pageserver_az_not_null/up.sql diff --git a/Dockerfile b/Dockerfile index d3d12330c6..1efedfa9bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -87,6 +87,7 @@ RUN mkdir -p /data/.neon/ && \ "pg_distrib_dir='/usr/local/'\n" \ "listen_pg_addr='0.0.0.0:6400'\n" \ "listen_http_addr='0.0.0.0:9898'\n" \ + "availability_zone='local'\n" \ > /data/.neon/pageserver.toml && \ chown -R neon:neon /data/.neon diff --git a/control_plane/storcon_cli/src/main.rs b/control_plane/storcon_cli/src/main.rs index 5cce6cf3ae..2a81a3d825 100644 --- a/control_plane/storcon_cli/src/main.rs +++ b/control_plane/storcon_cli/src/main.rs @@ -336,7 +336,7 @@ async fn main() -> anyhow::Result<()> { listen_pg_port, listen_http_addr, listen_http_port, - availability_zone_id: Some(availability_zone_id), + availability_zone_id, }), ) .await?; diff --git a/libs/pageserver_api/src/controller_api.rs b/libs/pageserver_api/src/controller_api.rs index 345abd69b6..6fb5a9a139 100644 --- a/libs/pageserver_api/src/controller_api.rs +++ b/libs/pageserver_api/src/controller_api.rs @@ -57,7 +57,7 @@ pub struct NodeRegisterRequest { pub listen_http_addr: String, pub listen_http_port: u16, - pub availability_zone_id: Option, + pub availability_zone_id: String, } #[derive(Serialize, Deserialize)] diff --git a/pageserver/src/control_plane_client.rs b/pageserver/src/control_plane_client.rs index 56a536c387..f6d1c35a8c 100644 --- a/pageserver/src/control_plane_client.rs +++ b/pageserver/src/control_plane_client.rs @@ -141,10 +141,24 @@ impl ControlPlaneGenerationsApi for ControlPlaneClient { m.other ); - let az_id = m - .other - .get("availability_zone_id") - .and_then(|jv| jv.as_str().map(|str| str.to_owned())); + let az_id = { + let az_id_from_metadata = m + .other + .get("availability_zone_id") + .and_then(|jv| jv.as_str().map(|str| str.to_owned())); + + match az_id_from_metadata { + Some(az_id) => Some(az_id), + None => { + tracing::warn!("metadata.json does not contain an 'availability_zone_id' field"); + conf.availability_zone.clone() + } + } + }; + + if az_id.is_none() { + panic!("Availablity zone id could not be inferred from metadata.json or pageserver config"); + } Some(NodeRegisterRequest { node_id: conf.id, @@ -152,7 +166,7 @@ impl ControlPlaneGenerationsApi for ControlPlaneClient { listen_pg_port: m.postgres_port, listen_http_addr: m.http_host, listen_http_port: m.http_port, - availability_zone_id: az_id, + availability_zone_id: az_id.expect("Checked above"), }) } Err(e) => { diff --git a/storage_controller/migrations/2024-08-28-150530_pageserver_az_not_null/down.sql b/storage_controller/migrations/2024-08-28-150530_pageserver_az_not_null/down.sql new file mode 100644 index 0000000000..4fcb928533 --- /dev/null +++ b/storage_controller/migrations/2024-08-28-150530_pageserver_az_not_null/down.sql @@ -0,0 +1 @@ +ALTER TABLE nodes ALTER availability_zone_id DROP NOT NULL; diff --git a/storage_controller/migrations/2024-08-28-150530_pageserver_az_not_null/up.sql b/storage_controller/migrations/2024-08-28-150530_pageserver_az_not_null/up.sql new file mode 100644 index 0000000000..c5b4534087 --- /dev/null +++ b/storage_controller/migrations/2024-08-28-150530_pageserver_az_not_null/up.sql @@ -0,0 +1 @@ +ALTER TABLE nodes ALTER availability_zone_id SET NOT NULL; diff --git a/storage_controller/src/node.rs b/storage_controller/src/node.rs index 73cecc491d..cb9ce10d23 100644 --- a/storage_controller/src/node.rs +++ b/storage_controller/src/node.rs @@ -36,7 +36,7 @@ pub(crate) struct Node { listen_pg_addr: String, listen_pg_port: u16, - availability_zone_id: Option, + availability_zone_id: String, // This cancellation token means "stop any RPCs in flight to this node, and don't start // any more". It is not related to process shutdown. @@ -63,8 +63,9 @@ impl Node { self.id } - pub(crate) fn get_availability_zone_id(&self) -> Option<&str> { - self.availability_zone_id.as_deref() + #[allow(unused)] + pub(crate) fn get_availability_zone_id(&self) -> &str { + self.availability_zone_id.as_str() } pub(crate) fn get_scheduling(&self) -> NodeSchedulingPolicy { @@ -78,22 +79,12 @@ impl Node { /// Does this registration request match `self`? This is used when deciding whether a registration /// request should be allowed to update an existing record with the same node ID. pub(crate) fn registration_match(&self, register_req: &NodeRegisterRequest) -> bool { - let az_ids_match = { - match ( - self.availability_zone_id.as_deref(), - register_req.availability_zone_id.as_deref(), - ) { - (Some(current_az), Some(register_req_az)) => current_az == register_req_az, - _ => true, - } - }; - - az_ids_match - && self.id == register_req.node_id + self.id == register_req.node_id && self.listen_http_addr == register_req.listen_http_addr && self.listen_http_port == register_req.listen_http_port && self.listen_pg_addr == register_req.listen_pg_addr && self.listen_pg_port == register_req.listen_pg_port + && self.availability_zone_id == register_req.availability_zone_id } /// For a shard located on this node, populate a response object @@ -190,7 +181,7 @@ impl Node { listen_http_port: u16, listen_pg_addr: String, listen_pg_port: u16, - availability_zone_id: Option, + availability_zone_id: String, ) -> Self { Self { id, diff --git a/storage_controller/src/persistence.rs b/storage_controller/src/persistence.rs index e801289752..6df05ebd13 100644 --- a/storage_controller/src/persistence.rs +++ b/storage_controller/src/persistence.rs @@ -105,7 +105,6 @@ pub(crate) enum DatabaseOperation { ListMetadataHealthOutdated, GetLeader, UpdateLeader, - SetNodeAzId, } #[must_use] @@ -325,31 +324,6 @@ impl Persistence { } } - pub(crate) async fn set_node_availability_zone_id( - &self, - input_node_id: NodeId, - input_az_id: String, - ) -> DatabaseResult<()> { - use crate::schema::nodes::dsl::*; - let updated = self - .with_measured_conn(DatabaseOperation::SetNodeAzId, move |conn| { - let updated = diesel::update(nodes) - .filter(node_id.eq(input_node_id.0 as i64)) - .set((availability_zone_id.eq(input_az_id.clone()),)) - .execute(conn)?; - Ok(updated) - }) - .await?; - - if updated != 1 { - Err(DatabaseError::Logical(format!( - "Node {node_id:?} not found for setting az id", - ))) - } else { - Ok(()) - } - } - /// At startup, load the high level state for shards, such as their config + policy. This will /// be enriched at runtime with state discovered on pageservers. pub(crate) async fn list_tenant_shards(&self) -> DatabaseResult> { @@ -1110,7 +1084,7 @@ pub(crate) struct NodePersistence { pub(crate) listen_http_port: i32, pub(crate) listen_pg_addr: String, pub(crate) listen_pg_port: i32, - pub(crate) availability_zone_id: Option, + pub(crate) availability_zone_id: String, } /// Tenant metadata health status that are stored durably. diff --git a/storage_controller/src/scheduler.rs b/storage_controller/src/scheduler.rs index ef4da6861c..deb5f27226 100644 --- a/storage_controller/src/scheduler.rs +++ b/storage_controller/src/scheduler.rs @@ -528,7 +528,7 @@ pub(crate) mod test_utils { 80 + i as u16, format!("pghost-{i}"), 5432 + i as u16, - None, + "test-az".to_string(), ); node.set_availability(NodeAvailability::Active(test_utilization::simple(0, 0))); assert!(node.is_available()); diff --git a/storage_controller/src/schema.rs b/storage_controller/src/schema.rs index e0f515daea..93ab774b5f 100644 --- a/storage_controller/src/schema.rs +++ b/storage_controller/src/schema.rs @@ -25,7 +25,7 @@ diesel::table! { listen_http_port -> Int4, listen_pg_addr -> Varchar, listen_pg_port -> Int4, - availability_zone_id -> Nullable, + availability_zone_id -> Varchar, } } diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index ca416095bb..2911cd5ac4 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -1264,7 +1264,7 @@ impl Service { 123, "".to_string(), 123, - None, + "test_az".to_string(), ); scheduler.node_upsert(&node); @@ -4825,15 +4825,8 @@ impl Service { ) .await; - if register_req.availability_zone_id.is_none() { - tracing::warn!( - "Node {} registering without specific availability zone id", - register_req.node_id - ); - } - enum RegistrationStatus { - Matched(Node), + Matched, Mismatched, New, } @@ -4842,7 +4835,7 @@ impl Service { let locked = self.inner.read().unwrap(); if let Some(node) = locked.nodes.get(®ister_req.node_id) { if node.registration_match(®ister_req) { - RegistrationStatus::Matched(node.clone()) + RegistrationStatus::Matched } else { RegistrationStatus::Mismatched } @@ -4852,41 +4845,12 @@ impl Service { }; match registration_status { - RegistrationStatus::Matched(node) => { + RegistrationStatus::Matched => { tracing::info!( "Node {} re-registered with matching address", register_req.node_id ); - if node.get_availability_zone_id().is_none() { - if let Some(az_id) = register_req.availability_zone_id.clone() { - tracing::info!("Extracting availability zone id from registration request for node {}: {}", - register_req.node_id, az_id); - - // Persist to the database and update in memory state. See comment below - // on ordering. - self.persistence - .set_node_availability_zone_id(register_req.node_id, az_id) - .await?; - let node_with_az = Node::new( - register_req.node_id, - register_req.listen_http_addr, - register_req.listen_http_port, - register_req.listen_pg_addr, - register_req.listen_pg_port, - register_req.availability_zone_id, - ); - - let mut locked = self.inner.write().unwrap(); - let mut new_nodes = (*locked.nodes).clone(); - - locked.scheduler.node_upsert(&node_with_az); - new_nodes.insert(register_req.node_id, node_with_az); - - locked.nodes = Arc::new(new_nodes); - } - } - return Ok(()); } RegistrationStatus::Mismatched => { diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 2df45a7e0e..0c692ceb69 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -758,6 +758,9 @@ class NeonEnvBuilder: patch_script = "" for ps in self.env.pageservers: patch_script += f"UPDATE nodes SET listen_http_port={ps.service_port.http}, listen_pg_port={ps.service_port.pg} WHERE node_id = '{ps.id}';" + # This is a temporary to get the backward compat test happy + # since the compat snapshot was generated with an older version of neon local + patch_script += f"UPDATE nodes SET availability_zone_id='{ps.az_id}' WHERE node_id = '{ps.id}' AND availability_zone_id IS NULL;" patch_script_path.write_text(patch_script) # Update the config with info about tenants and timelines From cf11c8ab6aa234b59354425116da98d58fa1826d Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 6 Sep 2024 10:52:29 +0200 Subject: [PATCH 30/45] update svg_fmt to 0.4.3 (#8930) Audited ``` diff -r -u ~/.cargo/registry/src/index.crates.io-6f17d22bba15001f/svg_fmt-0.4.{2,3} ``` fixes https://github.com/neondatabase/neon/issues/7763 --- Cargo.lock | 5 +++-- Cargo.toml | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 91917d5351..3f2787f15b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6094,8 +6094,9 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "svg_fmt" -version = "0.4.2" -source = "git+https://github.com/nical/rust_debug?rev=28a7d96eecff2f28e75b1ea09f2d499a60d0e3b4#28a7d96eecff2f28e75b1ea09f2d499a60d0e3b4" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20e16a0f46cf5fd675563ef54f26e83e20f2366bcf027bcb3cc3ed2b98aaf2ca" [[package]] name = "syn" diff --git a/Cargo.toml b/Cargo.toml index 4fea3e8d80..2415337110 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -161,8 +161,7 @@ socket2 = "0.5" strum = "0.24" strum_macros = "0.24" "subtle" = "2.5.0" -# Our PR https://github.com/nical/rust_debug/pull/4 has been merged but no new version released yet -svg_fmt = { git = "https://github.com/nical/rust_debug", rev = "28a7d96eecff2f28e75b1ea09f2d499a60d0e3b4" } +svg_fmt = "0.4.3" sync_wrapper = "0.1.2" tar = "0.4" task-local-extensions = "0.1.4" From 06e840b884c242550e2a5ad0e72bfa762bce1709 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 6 Sep 2024 10:58:48 +0200 Subject: [PATCH 31/45] compact_level0_phase1: ignore access mode config, always do streaming-kmerge without validation (#8934) refs https://github.com/neondatabase/neon/issues/8184 PR https://github.com/neondatabase/infra/pull/1905 enabled streaming-kmerge without validation everywhere. It rolls into prod sooner or in the same release as this PR. --- libs/pageserver_api/src/config.rs | 43 +----- pageserver/src/bin/pageserver.rs | 1 - pageserver/src/config.rs | 26 ++-- pageserver/src/tenant/timeline/compaction.rs | 139 +------------------ 4 files changed, 22 insertions(+), 187 deletions(-) diff --git a/libs/pageserver_api/src/config.rs b/libs/pageserver_api/src/config.rs index b2662c562a..1194ee93ef 100644 --- a/libs/pageserver_api/src/config.rs +++ b/libs/pageserver_api/src/config.rs @@ -104,7 +104,9 @@ pub struct ConfigToml { pub image_compression: ImageCompressionAlgorithm, pub ephemeral_bytes_per_memory_kb: usize, pub l0_flush: Option, - pub compact_level0_phase1_value_access: CompactL0Phase1ValueAccess, + #[serde(skip_serializing)] + // TODO(https://github.com/neondatabase/neon/issues/8184): remove after this field is removed from all pageserver.toml's + pub compact_level0_phase1_value_access: serde::de::IgnoredAny, pub virtual_file_direct_io: crate::models::virtual_file::DirectIoMode, pub io_buffer_alignment: usize, } @@ -209,43 +211,6 @@ pub enum GetImpl { #[serde(transparent)] pub struct MaxVectoredReadBytes(pub NonZeroUsize); -#[derive(Debug, PartialEq, Eq, Clone, serde::Deserialize, serde::Serialize)] -#[serde(tag = "mode", rename_all = "kebab-case", deny_unknown_fields)] -pub enum CompactL0Phase1ValueAccess { - /// The old way. - PageCachedBlobIo, - /// The new way. - StreamingKmerge { - /// If set, we run both the old way and the new way, validate that - /// they are identical (=> [`CompactL0BypassPageCacheValidation`]), - /// and if the validation fails, - /// - in tests: fail them with a panic or - /// - in prod, log a rate-limited warning and use the old way's results. - /// - /// If not set, we only run the new way and trust its results. - validate: Option, - }, -} - -/// See [`CompactL0Phase1ValueAccess::StreamingKmerge`]. -#[derive(Debug, PartialEq, Eq, Clone, serde::Deserialize, serde::Serialize)] -#[serde(rename_all = "kebab-case")] -pub enum CompactL0BypassPageCacheValidation { - /// Validate that the series of (key, lsn) pairs are the same. - KeyLsn, - /// Validate that the entire output of old and new way is identical. - KeyLsnValue, -} - -impl Default for CompactL0Phase1ValueAccess { - fn default() -> Self { - CompactL0Phase1ValueAccess::StreamingKmerge { - // TODO(https://github.com/neondatabase/neon/issues/8184): change to None once confident - validate: Some(CompactL0BypassPageCacheValidation::KeyLsnValue), - } - } -} - /// A tenant's calcuated configuration, which is the result of merging a /// tenant's TenantConfOpt with the global TenantConf from PageServerConf. /// @@ -452,7 +417,7 @@ impl Default for ConfigToml { image_compression: (DEFAULT_IMAGE_COMPRESSION), ephemeral_bytes_per_memory_kb: (DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB), l0_flush: None, - compact_level0_phase1_value_access: CompactL0Phase1ValueAccess::default(), + compact_level0_phase1_value_access: Default::default(), virtual_file_direct_io: crate::models::virtual_file::DirectIoMode::default(), io_buffer_alignment: DEFAULT_IO_BUFFER_ALIGNMENT, diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 2c60e8d7d1..59194ab4bd 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -125,7 +125,6 @@ fn main() -> anyhow::Result<()> { // after setting up logging, log the effective IO engine choice and read path implementations info!(?conf.virtual_file_io_engine, "starting with virtual_file IO engine"); info!(?conf.virtual_file_direct_io, "starting with virtual_file Direct IO settings"); - info!(?conf.compact_level0_phase1_value_access, "starting with setting for compact_level0_phase1_value_access"); info!(?conf.io_buffer_alignment, "starting with setting for IO buffer alignment"); // The tenants directory contains all the pageserver local disk state. diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index c159b66905..4e68e276d3 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -174,10 +174,6 @@ pub struct PageServerConf { pub l0_flush: crate::l0_flush::L0FlushConfig, - /// This flag is temporary and will be removed after gradual rollout. - /// See . - pub compact_level0_phase1_value_access: pageserver_api::config::CompactL0Phase1ValueAccess, - /// Direct IO settings pub virtual_file_direct_io: virtual_file::DirectIoMode, @@ -338,7 +334,7 @@ impl PageServerConf { max_vectored_read_bytes, image_compression, ephemeral_bytes_per_memory_kb, - compact_level0_phase1_value_access, + compact_level0_phase1_value_access: _, l0_flush, virtual_file_direct_io, concurrent_tenant_warmup, @@ -383,7 +379,6 @@ impl PageServerConf { max_vectored_read_bytes, image_compression, ephemeral_bytes_per_memory_kb, - compact_level0_phase1_value_access, virtual_file_direct_io, io_buffer_alignment, @@ -561,6 +556,16 @@ mod tests { .expect("parse_and_validate"); } + #[test] + fn test_compactl0_phase1_access_mode_is_ignored_silently() { + let input = indoc::indoc! {r#" + [compact_level0_phase1_value_access] + mode = "streaming-kmerge" + validate = "key-lsn-value" + "#}; + toml_edit::de::from_str::(input).unwrap(); + } + /// If there's a typo in the pageserver config, we'd rather catch that typo /// and fail pageserver startup than silently ignoring the typo, leaving whoever /// made it in the believe that their config change is effective. @@ -637,14 +642,5 @@ mod tests { // some_invalid_field = 23 // "#} // ); - - test!( - compact_level0_phase1_value_access, - indoc! {r#" - [compact_level0_phase1_value_access] - mode = "streaming-kmerge" - some_invalid_field = 23 - "#} - ); } } diff --git a/pageserver/src/tenant/timeline/compaction.rs b/pageserver/src/tenant/timeline/compaction.rs index 6b9c8386f7..a87b502cd6 100644 --- a/pageserver/src/tenant/timeline/compaction.rs +++ b/pageserver/src/tenant/timeline/compaction.rs @@ -19,7 +19,6 @@ use bytes::Bytes; use enumset::EnumSet; use fail::fail_point; use itertools::Itertools; -use pageserver_api::config::{CompactL0BypassPageCacheValidation, CompactL0Phase1ValueAccess}; use pageserver_api::key::KEY_SIZE; use pageserver_api::keyspace::ShardedRange; use pageserver_api::shard::{ShardCount, ShardIdentity, TenantShardId}; @@ -912,137 +911,13 @@ impl Timeline { // we're compacting, in key, LSN order. // If there's both a Value::Image and Value::WalRecord for the same (key,lsn), // then the Value::Image is ordered before Value::WalRecord. - // - // TODO(https://github.com/neondatabase/neon/issues/8184): remove the page cached blob_io - // option and validation code once we've reached confidence. - enum AllValuesIter<'a> { - PageCachedBlobIo { - all_keys_iter: VecIter<'a>, - }, - StreamingKmergeBypassingPageCache { - merge_iter: MergeIterator<'a>, - }, - ValidatingStreamingKmergeBypassingPageCache { - mode: CompactL0BypassPageCacheValidation, - merge_iter: MergeIterator<'a>, - all_keys_iter: VecIter<'a>, - }, - } - type VecIter<'a> = std::slice::Iter<'a, DeltaEntry<'a>>; // TODO: distinguished lifetimes - impl AllValuesIter<'_> { - async fn next_all_keys_iter( - iter: &mut VecIter<'_>, - ctx: &RequestContext, - ) -> anyhow::Result> { - let Some(DeltaEntry { - key, - lsn, - val: value_ref, - .. - }) = iter.next() - else { - return Ok(None); - }; - let value = value_ref.load(ctx).await?; - Ok(Some((*key, *lsn, value))) - } - async fn next( - &mut self, - ctx: &RequestContext, - ) -> anyhow::Result> { - match self { - AllValuesIter::PageCachedBlobIo { all_keys_iter: iter } => { - Self::next_all_keys_iter(iter, ctx).await - } - AllValuesIter::StreamingKmergeBypassingPageCache { merge_iter } => merge_iter.next().await, - AllValuesIter::ValidatingStreamingKmergeBypassingPageCache { mode, merge_iter, all_keys_iter } => async { - // advance both iterators - let all_keys_iter_item = Self::next_all_keys_iter(all_keys_iter, ctx).await; - let merge_iter_item = merge_iter.next().await; - // compare results & log warnings as needed - macro_rules! rate_limited_warn { - ($($arg:tt)*) => {{ - if cfg!(debug_assertions) || cfg!(feature = "testing") { - warn!($($arg)*); - panic!("CompactL0BypassPageCacheValidation failure, check logs"); - } - use once_cell::sync::Lazy; - use utils::rate_limit::RateLimit; - use std::sync::Mutex; - use std::time::Duration; - static LOGGED: Lazy> = - Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10)))); - let mut rate_limit = LOGGED.lock().unwrap(); - rate_limit.call(|| { - warn!($($arg)*); - }); - }} - } - match (&all_keys_iter_item, &merge_iter_item) { - (Err(_), Err(_)) => { - // don't bother asserting equivality of the errors - } - (Err(all_keys), Ok(merge)) => { - rate_limited_warn!(?merge, "all_keys_iter returned an error where merge did not: {all_keys:?}"); - }, - (Ok(all_keys), Err(merge)) => { - rate_limited_warn!(?all_keys, "merge returned an error where all_keys_iter did not: {merge:?}"); - }, - (Ok(None), Ok(None)) => { } - (Ok(Some(all_keys)), Ok(None)) => { - rate_limited_warn!(?all_keys, "merge returned None where all_keys_iter returned Some"); - } - (Ok(None), Ok(Some(merge))) => { - rate_limited_warn!(?merge, "all_keys_iter returned None where merge returned Some"); - } - (Ok(Some((all_keys_key, all_keys_lsn, all_keys_value))), Ok(Some((merge_key, merge_lsn, merge_value)))) => { - match mode { - // TODO: in this mode, we still load the value from disk for both iterators, even though we only need the all_keys_iter one - CompactL0BypassPageCacheValidation::KeyLsn => { - let all_keys = (all_keys_key, all_keys_lsn); - let merge = (merge_key, merge_lsn); - if all_keys != merge { - rate_limited_warn!(?all_keys, ?merge, "merge returned a different (Key,LSN) than all_keys_iter"); - } - } - CompactL0BypassPageCacheValidation::KeyLsnValue => { - let all_keys = (all_keys_key, all_keys_lsn, all_keys_value); - let merge = (merge_key, merge_lsn, merge_value); - if all_keys != merge { - rate_limited_warn!(?all_keys, ?merge, "merge returned a different (Key,LSN,Value) than all_keys_iter"); - } - } - } - } - } - // in case of mismatch, trust the legacy all_keys_iter_item - all_keys_iter_item - }.instrument(info_span!("next")).await - } - } - } - let mut all_values_iter = match &self.conf.compact_level0_phase1_value_access { - CompactL0Phase1ValueAccess::PageCachedBlobIo => AllValuesIter::PageCachedBlobIo { - all_keys_iter: all_keys.iter(), - }, - CompactL0Phase1ValueAccess::StreamingKmerge { validate } => { - let merge_iter = { - let mut deltas = Vec::with_capacity(deltas_to_compact.len()); - for l in deltas_to_compact.iter() { - let l = l.get_as_delta(ctx).await.map_err(CompactionError::Other)?; - deltas.push(l); - } - MergeIterator::create(&deltas, &[], ctx) - }; - match validate { - None => AllValuesIter::StreamingKmergeBypassingPageCache { merge_iter }, - Some(validate) => AllValuesIter::ValidatingStreamingKmergeBypassingPageCache { - mode: validate.clone(), - merge_iter, - all_keys_iter: all_keys.iter(), - }, - } + let mut all_values_iter = { + let mut deltas = Vec::with_capacity(deltas_to_compact.len()); + for l in deltas_to_compact.iter() { + let l = l.get_as_delta(ctx).await.map_err(CompactionError::Other)?; + deltas.push(l); } + MergeIterator::create(&deltas, &[], ctx) }; // This iterator walks through all keys and is needed to calculate size used by each key @@ -1119,7 +994,7 @@ impl Timeline { let mut keys = 0; while let Some((key, lsn, value)) = all_values_iter - .next(ctx) + .next() .await .map_err(CompactionError::Other)? { From a1323231bc65539f55eb1bfd341fb65d06d0ed22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Fri, 6 Sep 2024 12:40:19 +0200 Subject: [PATCH 32/45] Update Rust to 1.81.0 (#8939) We keep the practice of keeping the compiler up to date, pointing to the latest release. This is done by many other projects in the Rust ecosystem as well. [Release notes](https://github.com/rust-lang/rust/blob/master/RELEASES.md#version-1810-2024-09-05). Prior update was in #8667 and #8518 --- Cargo.lock | 30 +++++-------------- Cargo.toml | 2 +- Dockerfile.build-tools | 2 +- libs/postgres_ffi/build.rs | 2 +- libs/walproposer/build.rs | 21 ++++++++++--- libs/walproposer/src/api_bindings.rs | 10 +++---- .../tenant/remote_timeline_client/download.rs | 3 +- proxy/src/console/provider/neon.rs | 5 +--- rust-toolchain.toml | 2 +- safekeeper/src/send_wal.rs | 5 ++-- workspace_hack/Cargo.toml | 6 ++-- 11 files changed, 42 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f2787f15b..634af67198 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -915,25 +915,22 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.65.1" +version = "0.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.1", "cexpr", "clang-sys", - "lazy_static", - "lazycell", + "itertools 0.12.1", "log", - "peeking_take_while", - "prettyplease 0.2.6", + "prettyplease 0.2.17", "proc-macro2", "quote", "regex", "rustc-hash", "shlex", "syn 2.0.52", - "which", ] [[package]] @@ -2949,12 +2946,6 @@ dependencies = [ "spin 0.5.2", ] -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "libc" version = "0.2.150" @@ -3977,12 +3968,6 @@ dependencies = [ "sha2", ] -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "pem" version = "3.0.3" @@ -4280,9 +4265,9 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.6" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b69d39aab54d069e7f2fe8cb970493e7834601ca2d8c65fd7bbd183578080d1" +checksum = "8d3928fb5db768cb86f891ff014f0144589297e3c6a1aba6ed7cecfdace270c7" dependencies = [ "proc-macro2", "syn 2.0.52", @@ -7628,6 +7613,7 @@ dependencies = [ "hyper 0.14.26", "indexmap 1.9.3", "itertools 0.10.5", + "itertools 0.12.1", "lazy_static", "libc", "log", diff --git a/Cargo.toml b/Cargo.toml index 2415337110..5045ee0d4d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,7 @@ aws-types = "1.2.0" axum = { version = "0.6.20", features = ["ws"] } base64 = "0.13.0" bincode = "1.3" -bindgen = "0.65" +bindgen = "0.70" bit_field = "0.10.2" bstr = "1.0" byteorder = "1.4" diff --git a/Dockerfile.build-tools b/Dockerfile.build-tools index d6beb61369..a9cbed85fb 100644 --- a/Dockerfile.build-tools +++ b/Dockerfile.build-tools @@ -192,7 +192,7 @@ WORKDIR /home/nonroot # Rust # Please keep the version of llvm (installed above) in sync with rust llvm (`rustc --version --verbose | grep LLVM`) -ENV RUSTC_VERSION=1.80.1 +ENV RUSTC_VERSION=1.81.0 ENV RUSTUP_HOME="/home/nonroot/.rustup" ENV PATH="/home/nonroot/.cargo/bin:${PATH}" ARG RUSTFILT_VERSION=0.2.1 diff --git a/libs/postgres_ffi/build.rs b/libs/postgres_ffi/build.rs index 370d9e9a6f..d3e3ce648f 100644 --- a/libs/postgres_ffi/build.rs +++ b/libs/postgres_ffi/build.rs @@ -14,7 +14,7 @@ impl ParseCallbacks for PostgresFfiCallbacks { fn include_file(&self, filename: &str) { // This does the equivalent of passing bindgen::CargoCallbacks // to the builder .parse_callbacks() method. - let cargo_callbacks = bindgen::CargoCallbacks; + let cargo_callbacks = bindgen::CargoCallbacks::new(); cargo_callbacks.include_file(filename) } diff --git a/libs/walproposer/build.rs b/libs/walproposer/build.rs index 7bb077062b..28547f52bf 100644 --- a/libs/walproposer/build.rs +++ b/libs/walproposer/build.rs @@ -4,7 +4,6 @@ use std::{env, path::PathBuf, process::Command}; use anyhow::{anyhow, Context}; -use bindgen::CargoCallbacks; fn main() -> anyhow::Result<()> { // Tell cargo to invalidate the built crate whenever the wrapper changes @@ -64,16 +63,25 @@ fn main() -> anyhow::Result<()> { .map_err(|s| anyhow!("Bad postgres server path {s:?}"))? }; + let unwind_abi_functions = [ + "log_internal", + "recovery_download", + "start_streaming", + "finish_sync_safekeepers", + "wait_event_set", + "WalProposerStart", + ]; + // The bindgen::Builder is the main entry point // to bindgen, and lets you build up options for // the resulting bindings. - let bindings = bindgen::Builder::default() + let mut builder = bindgen::Builder::default() // The input header we would like to generate // bindings for. .header("bindgen_deps.h") // Tell cargo to invalidate the built crate whenever any of the // included header files changed. - .parse_callbacks(Box::new(CargoCallbacks)) + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) .allowlist_type("WalProposer") .allowlist_type("WalProposerConfig") .allowlist_type("walproposer_api") @@ -105,7 +113,12 @@ fn main() -> anyhow::Result<()> { .allowlist_var("WL_SOCKET_MASK") .clang_arg("-DWALPROPOSER_LIB") .clang_arg(format!("-I{pgxn_neon}")) - .clang_arg(format!("-I{inc_server_path}")) + .clang_arg(format!("-I{inc_server_path}")); + + for name in unwind_abi_functions { + builder = builder.override_abi(bindgen::Abi::CUnwind, name); + } + let bindings = builder // Finish the builder and generate the bindings. .generate() // Unwrap the Result and panic on failure. diff --git a/libs/walproposer/src/api_bindings.rs b/libs/walproposer/src/api_bindings.rs index bbc3663402..2fbea3fe45 100644 --- a/libs/walproposer/src/api_bindings.rs +++ b/libs/walproposer/src/api_bindings.rs @@ -33,7 +33,7 @@ extern "C" fn get_shmem_state(wp: *mut WalProposer) -> *mut WalproposerShmemStat } } -extern "C" fn start_streaming(wp: *mut WalProposer, startpos: XLogRecPtr) { +extern "C-unwind" fn start_streaming(wp: *mut WalProposer, startpos: XLogRecPtr) { unsafe { let callback_data = (*(*wp).config).callback_data; let api = callback_data as *mut Box; @@ -187,7 +187,7 @@ extern "C" fn conn_blocking_write( } } -extern "C" fn recovery_download(wp: *mut WalProposer, sk: *mut Safekeeper) -> bool { +extern "C-unwind" fn recovery_download(wp: *mut WalProposer, sk: *mut Safekeeper) -> bool { unsafe { let callback_data = (*(*(*sk).wp).config).callback_data; let api = callback_data as *mut Box; @@ -272,7 +272,7 @@ extern "C" fn rm_safekeeper_event_set(sk: *mut Safekeeper) { } } -extern "C" fn wait_event_set( +extern "C-unwind" fn wait_event_set( wp: *mut WalProposer, timeout: ::std::os::raw::c_long, event_sk: *mut *mut Safekeeper, @@ -324,7 +324,7 @@ extern "C" fn get_redo_start_lsn(wp: *mut WalProposer) -> XLogRecPtr { } } -extern "C" fn finish_sync_safekeepers(wp: *mut WalProposer, lsn: XLogRecPtr) { +extern "C-unwind" fn finish_sync_safekeepers(wp: *mut WalProposer, lsn: XLogRecPtr) { unsafe { let callback_data = (*(*wp).config).callback_data; let api = callback_data as *mut Box; @@ -340,7 +340,7 @@ extern "C" fn process_safekeeper_feedback(wp: *mut WalProposer, sk: *mut Safekee } } -extern "C" fn log_internal( +extern "C-unwind" fn log_internal( wp: *mut WalProposer, level: ::std::os::raw::c_int, line: *const ::std::os::raw::c_char, diff --git a/pageserver/src/tenant/remote_timeline_client/download.rs b/pageserver/src/tenant/remote_timeline_client/download.rs index d9725ad756..9fbe2f0da5 100644 --- a/pageserver/src/tenant/remote_timeline_client/download.rs +++ b/pageserver/src/tenant/remote_timeline_client/download.rs @@ -548,7 +548,7 @@ pub(crate) async fn download_initdb_tar_zst( cancel, ) .await - .map_err(|e| { + .inspect_err(|_e| { // Do a best-effort attempt at deleting the temporary file upon encountering an error. // We don't have async here nor do we want to pile on any extra errors. if let Err(e) = std::fs::remove_file(&temp_path) { @@ -556,7 +556,6 @@ pub(crate) async fn download_initdb_tar_zst( warn!("error deleting temporary file {temp_path}: {e}"); } } - e })?; Ok((temp_path, file)) diff --git a/proxy/src/console/provider/neon.rs b/proxy/src/console/provider/neon.rs index 33eda72e65..b004bf4ecf 100644 --- a/proxy/src/console/provider/neon.rs +++ b/proxy/src/console/provider/neon.rs @@ -38,10 +38,7 @@ impl Api { locks: &'static ApiLocks, wake_compute_endpoint_rate_limiter: Arc, ) -> Self { - let jwt = match std::env::var("NEON_PROXY_TO_CONTROLPLANE_TOKEN") { - Ok(v) => v, - Err(_) => String::new(), - }; + let jwt = std::env::var("NEON_PROXY_TO_CONTROLPLANE_TOKEN").unwrap_or_default(); Self { endpoint, caches, diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 368b8d300a..e78c4d6790 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] -channel = "1.80.1" +channel = "1.81.0" profile = "default" # The default profile includes rustc, rust-std, cargo, rust-docs, rustfmt and clippy. # https://rust-lang.github.io/rustup/concepts/profiles.html diff --git a/safekeeper/src/send_wal.rs b/safekeeper/src/send_wal.rs index 90b1604adb..6d677f405a 100644 --- a/safekeeper/src/send_wal.rs +++ b/safekeeper/src/send_wal.rs @@ -758,9 +758,8 @@ impl ReplyReader { // pq_sendint32(&reply_message, xmin); // pq_sendint32(&reply_message, xmin_epoch); // So it is two big endian 32-bit words in low endian order! - hs_feedback.xmin = (hs_feedback.xmin >> 32) | (hs_feedback.xmin << 32); - hs_feedback.catalog_xmin = - (hs_feedback.catalog_xmin >> 32) | (hs_feedback.catalog_xmin << 32); + hs_feedback.xmin = hs_feedback.xmin.rotate_left(32); + hs_feedback.catalog_xmin = hs_feedback.catalog_xmin.rotate_left(32); self.ws_guard .walsenders .record_hs_feedback(self.ws_guard.id, &hs_feedback); diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index 20693ad63d..3d2fa8c214 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -47,7 +47,8 @@ hex = { version = "0.4", features = ["serde"] } hmac = { version = "0.12", default-features = false, features = ["reset"] } hyper = { version = "0.14", features = ["full"] } indexmap = { version = "1", default-features = false, features = ["std"] } -itertools = { version = "0.10" } +itertools-5ef9efb8ec2df382 = { package = "itertools", version = "0.12", default-features = false, features = ["use_std"] } +itertools-93f6ce9d446188ac = { package = "itertools", version = "0.10" } lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } libc = { version = "0.2", features = ["extra_traits", "use_std"] } log = { version = "0.4", default-features = false, features = ["std"] } @@ -101,7 +102,8 @@ either = { version = "1" } getrandom = { version = "0.2", default-features = false, features = ["std"] } hashbrown = { version = "0.14", features = ["raw"] } indexmap = { version = "1", default-features = false, features = ["std"] } -itertools = { version = "0.10" } +itertools-5ef9efb8ec2df382 = { package = "itertools", version = "0.12", default-features = false, features = ["use_std"] } +itertools-93f6ce9d446188ac = { package = "itertools", version = "0.10" } lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } libc = { version = "0.2", features = ["extra_traits", "use_std"] } log = { version = "0.4", default-features = false, features = ["std"] } From e86fef05ddbc276170ec29d035d86d03e3ad4ec2 Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Fri, 6 Sep 2024 13:11:17 +0100 Subject: [PATCH 33/45] storcon: track preferred AZ for each tenant shard (#8937) ## Problem We want to do AZ aware scheduling, but don't have enough metadata. ## Summary of changes Introduce a `preferred_az_id` concept for each managed tenant shard. In a future PR, the scheduler will use this as a soft preference. The idea is to try and keep the shard attachments within the same AZ. Under the assumption that the compute was placed in the correct AZ, this reduces the chances of cross AZ trafic from between compute and PS. In terms of code changes we: 1. Add a new nullable `preferred_az_id` column to the `tenant_shards` table. Also include an in-memory counterpart. 2. Populate the preferred az on tenant creation and shard splits. 3. Add an endpoint which allows to bulk-set preferred AZs. (3) gives us the migration path. I'll write a script which queries the cplane db in the region and sets the preferred az of all shards with an active compute to the AZ of said compute. For shards without an active compute, I'll use the AZ of the currently attached pageserver since this is what cplane uses now to schedule computes. --- libs/pageserver_api/src/controller_api.rs | 15 +- .../down.sql | 1 + .../up.sql | 1 + storage_controller/src/http.rs | 21 +- storage_controller/src/persistence.rs | 33 ++ storage_controller/src/schema.rs | 1 + storage_controller/src/service.rs | 327 +++++++++++++----- storage_controller/src/tenant_shard.rs | 15 + test_runner/fixtures/neon_fixtures.py | 13 +- .../regress/test_storage_controller.py | 52 +++ 10 files changed, 384 insertions(+), 95 deletions(-) create mode 100644 storage_controller/migrations/2024-09-05-104500_tenant_shard_preferred_az/down.sql create mode 100644 storage_controller/migrations/2024-09-05-104500_tenant_shard_preferred_az/up.sql diff --git a/libs/pageserver_api/src/controller_api.rs b/libs/pageserver_api/src/controller_api.rs index 6fb5a9a139..94104af002 100644 --- a/libs/pageserver_api/src/controller_api.rs +++ b/libs/pageserver_api/src/controller_api.rs @@ -1,4 +1,4 @@ -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::str::FromStr; use std::time::{Duration, Instant}; @@ -74,6 +74,17 @@ pub struct TenantPolicyRequest { pub scheduling: Option, } +#[derive(Serialize, Deserialize)] +pub struct ShardsPreferredAzsRequest { + #[serde(flatten)] + pub preferred_az_ids: HashMap, +} + +#[derive(Serialize, Deserialize)] +pub struct ShardsPreferredAzsResponse { + pub updated: Vec, +} + #[derive(Serialize, Deserialize, Debug)] pub struct TenantLocateResponseShard { pub shard_id: TenantShardId, @@ -132,6 +143,8 @@ pub struct TenantDescribeResponseShard { pub is_splitting: bool, pub scheduling_policy: ShardSchedulingPolicy, + + pub preferred_az_id: Option, } /// Explicitly migrating a particular shard is a low level operation diff --git a/storage_controller/migrations/2024-09-05-104500_tenant_shard_preferred_az/down.sql b/storage_controller/migrations/2024-09-05-104500_tenant_shard_preferred_az/down.sql new file mode 100644 index 0000000000..127972a2e4 --- /dev/null +++ b/storage_controller/migrations/2024-09-05-104500_tenant_shard_preferred_az/down.sql @@ -0,0 +1 @@ +ALTER TABLE tenant_shards DROP preferred_az_id; diff --git a/storage_controller/migrations/2024-09-05-104500_tenant_shard_preferred_az/up.sql b/storage_controller/migrations/2024-09-05-104500_tenant_shard_preferred_az/up.sql new file mode 100644 index 0000000000..641a54feb2 --- /dev/null +++ b/storage_controller/migrations/2024-09-05-104500_tenant_shard_preferred_az/up.sql @@ -0,0 +1 @@ +ALTER TABLE tenant_shards ADD preferred_az_id VARCHAR; diff --git a/storage_controller/src/http.rs b/storage_controller/src/http.rs index 32882c201a..5d4d0460be 100644 --- a/storage_controller/src/http.rs +++ b/storage_controller/src/http.rs @@ -14,7 +14,7 @@ use metrics::{BuildInfo, NeonMetrics}; use pageserver_api::controller_api::{ MetadataHealthListOutdatedRequest, MetadataHealthListOutdatedResponse, MetadataHealthListUnhealthyResponse, MetadataHealthUpdateRequest, MetadataHealthUpdateResponse, - TenantCreateRequest, + ShardsPreferredAzsRequest, TenantCreateRequest, }; use pageserver_api::models::{ TenantConfigRequest, TenantLocationConfigRequest, TenantShardSplitRequest, @@ -688,6 +688,18 @@ async fn handle_tenant_update_policy(mut req: Request) -> Result) -> Result, ApiError> { + check_permissions(&req, Scope::Admin)?; + + let azs_req = json_request::(&mut req).await?; + let state = get_state(&req); + + json_response( + StatusCode::OK, + state.service.update_shards_preferred_azs(azs_req).await?, + ) +} + async fn handle_step_down(req: Request) -> Result, ApiError> { check_permissions(&req, Scope::Admin)?; @@ -1174,6 +1186,13 @@ pub fn make_router( RequestName("control_v1_tenant_policy"), ) }) + .put("/control/v1/preferred_azs", |r| { + named_request_span( + r, + handle_update_preferred_azs, + RequestName("control_v1_preferred_azs"), + ) + }) .put("/control/v1/step_down", |r| { named_request_span(r, handle_step_down, RequestName("control_v1_step_down")) }) diff --git a/storage_controller/src/persistence.rs b/storage_controller/src/persistence.rs index 6df05ebd13..1dc1040d96 100644 --- a/storage_controller/src/persistence.rs +++ b/storage_controller/src/persistence.rs @@ -105,6 +105,7 @@ pub(crate) enum DatabaseOperation { ListMetadataHealthOutdated, GetLeader, UpdateLeader, + SetPreferredAzs, } #[must_use] @@ -664,6 +665,33 @@ impl Persistence { Ok(()) } + pub(crate) async fn set_tenant_shard_preferred_azs( + &self, + preferred_azs: Vec<(TenantShardId, String)>, + ) -> DatabaseResult> { + use crate::schema::tenant_shards::dsl::*; + + self.with_measured_conn(DatabaseOperation::SetPreferredAzs, move |conn| { + let mut shards_updated = Vec::default(); + + for (tenant_shard_id, preferred_az) in preferred_azs.iter() { + let updated = diesel::update(tenant_shards) + .filter(tenant_id.eq(tenant_shard_id.tenant_id.to_string())) + .filter(shard_number.eq(tenant_shard_id.shard_number.0 as i32)) + .filter(shard_count.eq(tenant_shard_id.shard_count.literal() as i32)) + .set(preferred_az_id.eq(preferred_az)) + .execute(conn)?; + + if updated == 1 { + shards_updated.push((*tenant_shard_id, preferred_az.clone())); + } + } + + Ok(shards_updated) + }) + .await + } + pub(crate) async fn detach(&self, tenant_shard_id: TenantShardId) -> anyhow::Result<()> { use crate::schema::tenant_shards::dsl::*; self.with_measured_conn(DatabaseOperation::Detach, move |conn| { @@ -1050,6 +1078,11 @@ pub(crate) struct TenantShardPersistence { pub(crate) config: String, #[serde(default)] pub(crate) scheduling_policy: String, + + // Hint that we should attempt to schedule this tenant shard the given + // availability zone in order to minimise the chances of cross-AZ communication + // with compute. + pub(crate) preferred_az_id: Option, } impl TenantShardPersistence { diff --git a/storage_controller/src/schema.rs b/storage_controller/src/schema.rs index 93ab774b5f..1717a9369d 100644 --- a/storage_controller/src/schema.rs +++ b/storage_controller/src/schema.rs @@ -41,6 +41,7 @@ diesel::table! { splitting -> Int2, config -> Text, scheduling_policy -> Varchar, + preferred_az_id -> Nullable, } } diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 2911cd5ac4..324f864291 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -25,7 +25,7 @@ use crate::{ ShardGenerationState, TenantFilter, }, reconciler::{ReconcileError, ReconcileUnits, ReconcilerConfig, ReconcilerConfigBuilder}, - scheduler::{MaySchedule, ScheduleContext, ScheduleMode}, + scheduler::{MaySchedule, ScheduleContext, ScheduleError, ScheduleMode}, tenant_shard::{ MigrateAttachment, ReconcileNeeded, ReconcilerStatus, ScheduleOptimization, ScheduleOptimizationAction, @@ -41,10 +41,11 @@ use itertools::Itertools; use pageserver_api::{ controller_api::{ MetadataHealthRecord, MetadataHealthUpdateRequest, NodeAvailability, NodeRegisterRequest, - NodeSchedulingPolicy, PlacementPolicy, ShardSchedulingPolicy, TenantCreateRequest, - TenantCreateResponse, TenantCreateResponseShard, TenantDescribeResponse, - TenantDescribeResponseShard, TenantLocateResponse, TenantPolicyRequest, - TenantShardMigrateRequest, TenantShardMigrateResponse, + NodeSchedulingPolicy, PlacementPolicy, ShardSchedulingPolicy, ShardsPreferredAzsRequest, + ShardsPreferredAzsResponse, TenantCreateRequest, TenantCreateResponse, + TenantCreateResponseShard, TenantDescribeResponse, TenantDescribeResponseShard, + TenantLocateResponse, TenantPolicyRequest, TenantShardMigrateRequest, + TenantShardMigrateResponse, }, models::{ SecondaryProgress, TenantConfigRequest, TimelineArchivalConfigRequest, @@ -353,6 +354,12 @@ impl From for ApiError { } } +enum InitialShardScheduleOutcome { + Scheduled(TenantCreateResponseShard), + NotScheduled, + ShardScheduleError(ScheduleError), +} + pub struct Service { inner: Arc>, config: Config, @@ -1452,6 +1459,7 @@ impl Service { splitting: SplitState::default(), scheduling_policy: serde_json::to_string(&ShardSchedulingPolicy::default()) .unwrap(), + preferred_az_id: None, }; match self.persistence.insert_tenant_shards(vec![tsp]).await { @@ -2023,6 +2031,7 @@ impl Service { splitting: SplitState::default(), scheduling_policy: serde_json::to_string(&ShardSchedulingPolicy::default()) .unwrap(), + preferred_az_id: None, }) .collect(); @@ -2046,99 +2055,87 @@ impl Service { }; let mut schedule_context = ScheduleContext::default(); + let mut schedule_error = None; + let mut response_shards = Vec::new(); + for tenant_shard_id in create_ids { + tracing::info!("Creating shard {tenant_shard_id}..."); - let (waiters, response_shards) = { + let outcome = self + .do_initial_shard_scheduling( + tenant_shard_id, + initial_generation, + &create_req.shard_parameters, + create_req.config.clone(), + placement_policy.clone(), + &mut schedule_context, + ) + .await; + + match outcome { + InitialShardScheduleOutcome::Scheduled(resp) => response_shards.push(resp), + InitialShardScheduleOutcome::NotScheduled => {} + InitialShardScheduleOutcome::ShardScheduleError(err) => { + schedule_error = Some(err); + } + } + } + + let preferred_azs = { + let locked = self.inner.read().unwrap(); + response_shards + .iter() + .filter_map(|resp| { + let az_id = locked + .nodes + .get(&resp.node_id) + .map(|n| n.get_availability_zone_id().to_string())?; + + Some((resp.shard_id, az_id)) + }) + .collect::>() + }; + + // Note that we persist the preferred AZ for the new shards separately. + // In theory, we could "peek" the scheduler to determine where the shard will + // land, but the subsequent "real" call into the scheduler might select a different + // node. Hence, we do this awkward update to keep things consistent. + let updated = self + .persistence + .set_tenant_shard_preferred_azs(preferred_azs) + .await + .map_err(|err| { + ApiError::InternalServerError(anyhow::anyhow!( + "Failed to persist preferred az ids: {err}" + )) + })?; + + { let mut locked = self.inner.write().unwrap(); - let (nodes, tenants, scheduler) = locked.parts_mut(); - - let mut response_shards = Vec::new(); - let mut schcedule_error = None; - - for tenant_shard_id in create_ids { - tracing::info!("Creating shard {tenant_shard_id}..."); - - use std::collections::btree_map::Entry; - match tenants.entry(tenant_shard_id) { - Entry::Occupied(mut entry) => { - tracing::info!( - "Tenant shard {tenant_shard_id} already exists while creating" - ); - - // TODO: schedule() should take an anti-affinity expression that pushes - // attached and secondary locations (independently) away frorm those - // pageservers also holding a shard for this tenant. - - entry - .get_mut() - .schedule(scheduler, &mut schedule_context) - .map_err(|e| { - ApiError::Conflict(format!( - "Failed to schedule shard {tenant_shard_id}: {e}" - )) - })?; - - if let Some(node_id) = entry.get().intent.get_attached() { - let generation = entry - .get() - .generation - .expect("Generation is set when in attached mode"); - response_shards.push(TenantCreateResponseShard { - shard_id: tenant_shard_id, - node_id: *node_id, - generation: generation.into().unwrap(), - }); - } - - continue; - } - Entry::Vacant(entry) => { - let state = entry.insert(TenantShard::new( - tenant_shard_id, - ShardIdentity::from_params( - tenant_shard_id.shard_number, - &create_req.shard_parameters, - ), - placement_policy.clone(), - )); - - state.generation = initial_generation; - state.config = create_req.config.clone(); - if let Err(e) = state.schedule(scheduler, &mut schedule_context) { - schcedule_error = Some(e); - } - - // Only include shards in result if we are attaching: the purpose - // of the response is to tell the caller where the shards are attached. - if let Some(node_id) = state.intent.get_attached() { - let generation = state - .generation - .expect("Generation is set when in attached mode"); - response_shards.push(TenantCreateResponseShard { - shard_id: tenant_shard_id, - node_id: *node_id, - generation: generation.into().unwrap(), - }); - } - } - }; + for (tid, az_id) in updated { + if let Some(shard) = locked.tenants.get_mut(&tid) { + shard.set_preferred_az(az_id); + } } + } - // If we failed to schedule shards, then they are still created in the controller, - // but we return an error to the requester to avoid a silent failure when someone - // tries to e.g. create a tenant whose placement policy requires more nodes than - // are present in the system. We do this here rather than in the above loop, to - // avoid situations where we only create a subset of shards in the tenant. - if let Some(e) = schcedule_error { - return Err(ApiError::Conflict(format!( - "Failed to schedule shard(s): {e}" - ))); - } + // If we failed to schedule shards, then they are still created in the controller, + // but we return an error to the requester to avoid a silent failure when someone + // tries to e.g. create a tenant whose placement policy requires more nodes than + // are present in the system. We do this here rather than in the above loop, to + // avoid situations where we only create a subset of shards in the tenant. + if let Some(e) = schedule_error { + return Err(ApiError::Conflict(format!( + "Failed to schedule shard(s): {e}" + ))); + } - let waiters = tenants + let waiters = { + let mut locked = self.inner.write().unwrap(); + let (nodes, tenants, _scheduler) = locked.parts_mut(); + tenants .range_mut(TenantShardId::tenant_range(tenant_id)) .filter_map(|(_shard_id, shard)| self.maybe_reconcile_shard(shard, nodes)) - .collect::>(); - (waiters, response_shards) + .collect::>() }; Ok(( @@ -2149,6 +2146,78 @@ impl Service { )) } + /// Helper for tenant creation that does the scheduling for an individual shard. Covers both the + /// case of a new tenant and a pre-existing one. + async fn do_initial_shard_scheduling( + &self, + tenant_shard_id: TenantShardId, + initial_generation: Option, + shard_params: &ShardParameters, + config: TenantConfig, + placement_policy: PlacementPolicy, + schedule_context: &mut ScheduleContext, + ) -> InitialShardScheduleOutcome { + let mut locked = self.inner.write().unwrap(); + let (_nodes, tenants, scheduler) = locked.parts_mut(); + + use std::collections::btree_map::Entry; + match tenants.entry(tenant_shard_id) { + Entry::Occupied(mut entry) => { + tracing::info!("Tenant shard {tenant_shard_id} already exists while creating"); + + // TODO: schedule() should take an anti-affinity expression that pushes + // attached and secondary locations (independently) away frorm those + // pageservers also holding a shard for this tenant. + + if let Err(err) = entry.get_mut().schedule(scheduler, schedule_context) { + return InitialShardScheduleOutcome::ShardScheduleError(err); + } + + if let Some(node_id) = entry.get().intent.get_attached() { + let generation = entry + .get() + .generation + .expect("Generation is set when in attached mode"); + InitialShardScheduleOutcome::Scheduled(TenantCreateResponseShard { + shard_id: tenant_shard_id, + node_id: *node_id, + generation: generation.into().unwrap(), + }) + } else { + InitialShardScheduleOutcome::NotScheduled + } + } + Entry::Vacant(entry) => { + let state = entry.insert(TenantShard::new( + tenant_shard_id, + ShardIdentity::from_params(tenant_shard_id.shard_number, shard_params), + placement_policy, + )); + + state.generation = initial_generation; + state.config = config; + if let Err(e) = state.schedule(scheduler, schedule_context) { + return InitialShardScheduleOutcome::ShardScheduleError(e); + } + + // Only include shards in result if we are attaching: the purpose + // of the response is to tell the caller where the shards are attached. + if let Some(node_id) = state.intent.get_attached() { + let generation = state + .generation + .expect("Generation is set when in attached mode"); + InitialShardScheduleOutcome::Scheduled(TenantCreateResponseShard { + shard_id: tenant_shard_id, + node_id: *node_id, + generation: generation.into().unwrap(), + }) + } else { + InitialShardScheduleOutcome::NotScheduled + } + } + } + } + /// Helper for functions that reconcile a number of shards, and would like to do a timeout-bounded /// wait for reconciliation to complete before responding. async fn await_waiters( @@ -3511,6 +3580,7 @@ impl Service { is_pending_compute_notification: shard.pending_compute_notification, is_splitting: matches!(shard.splitting, SplitState::Splitting), scheduling_policy: *shard.get_scheduling_policy(), + preferred_az_id: shard.preferred_az().map(ToString::to_string), }) } @@ -4214,9 +4284,10 @@ impl Service { config: serde_json::to_string(&config).unwrap(), splitting: SplitState::Splitting, - // Scheduling policies do not carry through to children + // Scheduling policies and preferred AZ do not carry through to children scheduling_policy: serde_json::to_string(&ShardSchedulingPolicy::default()) .unwrap(), + preferred_az_id: None, }); } @@ -4336,6 +4407,47 @@ impl Service { let (response, child_locations, waiters) = self.tenant_shard_split_commit_inmem(tenant_id, new_shard_count, new_stripe_size); + // Now that we have scheduled the child shards, attempt to set their preferred AZ + // to that of the pageserver they've been attached on. + let preferred_azs = { + let locked = self.inner.read().unwrap(); + child_locations + .iter() + .filter_map(|(tid, node_id, _stripe_size)| { + let az_id = locked + .nodes + .get(node_id) + .map(|n| n.get_availability_zone_id().to_string())?; + + Some((*tid, az_id)) + }) + .collect::>() + }; + + let updated = self + .persistence + .set_tenant_shard_preferred_azs(preferred_azs) + .await + .map_err(|err| { + ApiError::InternalServerError(anyhow::anyhow!( + "Failed to persist preferred az ids: {err}" + )) + }); + + match updated { + Ok(updated) => { + let mut locked = self.inner.write().unwrap(); + for (tid, az_id) in updated { + if let Some(shard) = locked.tenants.get_mut(&tid) { + shard.set_preferred_az(az_id); + } + } + } + Err(err) => { + tracing::warn!("Failed to persist preferred AZs after split: {err}"); + } + } + // Send compute notifications for all the new shards let mut failed_notifications = Vec::new(); for (child_id, child_ps, stripe_size) in child_locations { @@ -6497,4 +6609,35 @@ impl Service { ) -> Result<(), DatabaseError> { self.persistence.safekeeper_upsert(record).await } + + pub(crate) async fn update_shards_preferred_azs( + &self, + req: ShardsPreferredAzsRequest, + ) -> Result { + let preferred_azs = req.preferred_az_ids.into_iter().collect::>(); + let updated = self + .persistence + .set_tenant_shard_preferred_azs(preferred_azs) + .await + .map_err(|err| { + ApiError::InternalServerError(anyhow::anyhow!( + "Failed to persist preferred AZs: {err}" + )) + })?; + + let mut updated_in_mem_and_db = Vec::default(); + + let mut locked = self.inner.write().unwrap(); + for (tid, az_id) in updated { + let shard = locked.tenants.get_mut(&tid); + if let Some(shard) = shard { + shard.set_preferred_az(az_id); + updated_in_mem_and_db.push(tid); + } + } + + Ok(ShardsPreferredAzsResponse { + updated: updated_in_mem_and_db, + }) + } } diff --git a/storage_controller/src/tenant_shard.rs b/storage_controller/src/tenant_shard.rs index 30723a3b36..cdb0633e2b 100644 --- a/storage_controller/src/tenant_shard.rs +++ b/storage_controller/src/tenant_shard.rs @@ -140,6 +140,10 @@ pub(crate) struct TenantShard { // Support/debug tool: if something is going wrong or flapping with scheduling, this may // be set to a non-active state to avoid making changes while the issue is fixed. scheduling_policy: ShardSchedulingPolicy, + + // We should attempt to schedule this shard in the provided AZ to + // decrease chances of cross-AZ compute. + preferred_az_id: Option, } #[derive(Default, Clone, Debug, Serialize)] @@ -463,6 +467,7 @@ impl TenantShard { last_error: Arc::default(), pending_compute_notification: false, scheduling_policy: ShardSchedulingPolicy::default(), + preferred_az_id: None, } } @@ -1297,6 +1302,7 @@ impl TenantShard { pending_compute_notification: false, delayed_reconcile: false, scheduling_policy: serde_json::from_str(&tsp.scheduling_policy).unwrap(), + preferred_az_id: tsp.preferred_az_id, }) } @@ -1312,8 +1318,17 @@ impl TenantShard { config: serde_json::to_string(&self.config).unwrap(), splitting: SplitState::default(), scheduling_policy: serde_json::to_string(&self.scheduling_policy).unwrap(), + preferred_az_id: self.preferred_az_id.clone(), } } + + pub(crate) fn preferred_az(&self) -> Option<&str> { + self.preferred_az_id.as_deref() + } + + pub(crate) fn set_preferred_az(&mut self, preferred_az_id: String) { + self.preferred_az_id = Some(preferred_az_id); + } } #[cfg(test)] diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 0c692ceb69..18fbbde637 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2560,7 +2560,7 @@ class NeonStorageController(MetricsGetter, LogUtils): def tenant_describe(self, tenant_id: TenantId): """ - :return: list of {"shard_id": "", "node_id": int, "listen_pg_addr": str, "listen_pg_port": int, "listen_http_addr: str, "listen_http_port: int} + :return: list of {"shard_id": "", "node_id": int, "listen_pg_addr": str, "listen_pg_port": int, "listen_http_addr: str, "listen_http_port: int, preferred_az_id: str} """ response = self.request( "GET", @@ -2886,6 +2886,17 @@ class NeonStorageController(MetricsGetter, LogUtils): return None raise e + def set_preferred_azs(self, preferred_azs: dict[TenantShardId, str]) -> list[TenantShardId]: + response = self.request( + "PUT", + f"{self.api}/control/v1/preferred_azs", + headers=self.headers(TokenScope.ADMIN), + json={str(tid): az for tid, az in preferred_azs.items()}, + ) + + response.raise_for_status() + return [TenantShardId.parse(tid) for tid in response.json()["updated"]] + def __enter__(self) -> "NeonStorageController": return self diff --git a/test_runner/regress/test_storage_controller.py b/test_runner/regress/test_storage_controller.py index 8da42294b0..92cd74eba5 100644 --- a/test_runner/regress/test_storage_controller.py +++ b/test_runner/regress/test_storage_controller.py @@ -2512,3 +2512,55 @@ def eq_safekeeper_records(a: dict[str, Any], b: dict[str, Any]) -> bool: del d[key] return compared[0] == compared[1] + + +@run_only_on_default_postgres("this is like a 'unit test' against storcon db") +def test_shard_preferred_azs(neon_env_builder: NeonEnvBuilder): + def assign_az(ps_cfg): + az = f"az-{ps_cfg['id']}" + ps_cfg["availability_zone"] = az + + neon_env_builder.pageserver_config_override = assign_az + neon_env_builder.num_pageservers = 2 + env = neon_env_builder.init_configs() + env.start() + + tids = [TenantId.generate() for _ in range(0, 3)] + for tid in tids: + env.storage_controller.tenant_create(tid) + + shards = env.storage_controller.tenant_describe(tid)["shards"] + assert len(shards) == 1 + attached_to = shards[0]["node_attached"] + expected_az = env.get_pageserver(attached_to).az_id + + assert shards[0]["preferred_az_id"] == expected_az + + updated = env.storage_controller.set_preferred_azs( + {TenantShardId(tid, 0, 0): "foo" for tid in tids} + ) + + assert set(updated) == set([TenantShardId(tid, 0, 0) for tid in tids]) + + for tid in tids: + shards = env.storage_controller.tenant_describe(tid)["shards"] + assert len(shards) == 1 + assert shards[0]["preferred_az_id"] == "foo" + + # Generate a layer to avoid shard split handling on ps from tripping + # up on debug assert. + timeline_id = TimelineId.generate() + env.neon_cli.create_timeline("bar", tids[0], timeline_id) + + workload = Workload(env, tids[0], timeline_id, branch_name="bar") + workload.init() + workload.write_rows(256) + workload.validate() + + env.storage_controller.tenant_shard_split(tids[0], shard_count=2) + shards = env.storage_controller.tenant_describe(tids[0])["shards"] + assert len(shards) == 2 + for shard in shards: + attached_to = shard["node_attached"] + expected_az = env.get_pageserver(attached_to).az_id + assert shard["preferred_az_id"] == expected_az From cbcd4058edb7a2c2bb3bfe1a6fc1ffb0d820b870 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Fri, 6 Sep 2024 14:33:52 +0200 Subject: [PATCH 34/45] Fix 1.82 clippy lint too_long_first_doc_paragraph (#8941) Addresses the 1.82 beta clippy lint `too_long_first_doc_paragraph` by adding newlines to the first sentence if it is short enough, and making a short first sentence if there is the need. --- compute_tools/src/pg_helpers.rs | 7 ++++--- libs/metrics/src/lib.rs | 1 + libs/pageserver_api/src/controller_api.rs | 2 ++ libs/pageserver_api/src/models.rs | 10 +++++++--- libs/postgres_backend/src/lib.rs | 6 ++++-- libs/postgres_connection/src/lib.rs | 1 + libs/remote_storage/src/lib.rs | 6 +++++- libs/tenant_size_model/src/lib.rs | 7 ++++--- libs/utils/src/circuit_breaker.rs | 6 ++++-- libs/utils/src/id.rs | 6 ++++-- libs/utils/src/lock_file.rs | 4 +++- libs/utils/src/pageserver_feedback.rs | 1 + libs/utils/src/poison.rs | 2 ++ libs/utils/src/shard.rs | 9 +++++---- libs/utils/src/simple_rcu.rs | 7 +++---- libs/utils/src/sync/heavier_once_cell.rs | 4 +++- libs/utils/src/vec_map.rs | 1 + libs/utils/src/yielding_loop.rs | 7 ++++--- pageserver/src/config.rs | 2 ++ pageserver/src/context.rs | 10 ++++++---- pageserver/src/pgdatadir_mapping.rs | 8 +++++--- pageserver/src/tenant.rs | 9 +++++---- pageserver/src/tenant/metadata.rs | 9 +++++---- pageserver/src/tenant/mgr.rs | 12 +++++++----- pageserver/src/tenant/remote_timeline_client.rs | 2 ++ .../src/tenant/remote_timeline_client/index.rs | 1 + pageserver/src/tenant/storage_layer.rs | 9 +++++---- pageserver/src/tenant/storage_layer/delta_layer.rs | 9 +++++---- pageserver/src/tenant/storage_layer/image_layer.rs | 8 +++++--- pageserver/src/tenant/storage_layer/layer_desc.rs | 6 ++++-- pageserver/src/tenant/storage_layer/layer_name.rs | 5 +++-- .../src/tenant/storage_layer/merge_iterator.rs | 8 +++++--- .../src/tenant/storage_layer/split_writer.rs | 14 ++++++++------ pageserver/src/tenant/vectored_blob_io.rs | 6 ++++-- pageserver/src/virtual_file.rs | 5 +++-- pageserver/src/walredo.rs | 11 +++++------ proxy/src/stream.rs | 1 + safekeeper/src/pull_timeline.rs | 1 + safekeeper/src/receive_wal.rs | 6 ++++-- safekeeper/src/state.rs | 8 +++++--- safekeeper/src/timeline.rs | 1 + safekeeper/src/timeline_eviction.rs | 8 +++++--- safekeeper/src/timeline_guard.rs | 4 +++- safekeeper/src/timeline_manager.rs | 1 + safekeeper/src/timelines_set.rs | 3 ++- safekeeper/src/wal_backup_partial.rs | 6 ++++-- safekeeper/src/wal_service.rs | 1 + storage_controller/src/service.rs | 4 +++- storage_scrubber/src/garbage.rs | 7 ++++--- storage_scrubber/src/metadata_stream.rs | 4 +++- storage_scrubber/src/pageserver_physical_gc.rs | 7 ++++--- 51 files changed, 180 insertions(+), 103 deletions(-) diff --git a/compute_tools/src/pg_helpers.rs b/compute_tools/src/pg_helpers.rs index 863fa9468f..b2dc265864 100644 --- a/compute_tools/src/pg_helpers.rs +++ b/compute_tools/src/pg_helpers.rs @@ -22,9 +22,10 @@ use compute_api::spec::{Database, GenericOption, GenericOptions, PgIdent, Role}; const POSTGRES_WAIT_TIMEOUT: Duration = Duration::from_millis(60 * 1000); // milliseconds -/// Escape a string for including it in a SQL literal. Wrapping the result -/// with `E'{}'` or `'{}'` is not required, as it returns a ready-to-use -/// SQL string literal, e.g. `'db'''` or `E'db\\'`. +/// Escape a string for including it in a SQL literal. +/// +/// Wrapping the result with `E'{}'` or `'{}'` is not required, +/// as it returns a ready-to-use SQL string literal, e.g. `'db'''` or `E'db\\'`. /// See /// for the original implementation. pub fn escape_literal(s: &str) -> String { diff --git a/libs/metrics/src/lib.rs b/libs/metrics/src/lib.rs index df000cd0fb..cd4526c089 100644 --- a/libs/metrics/src/lib.rs +++ b/libs/metrics/src/lib.rs @@ -68,6 +68,7 @@ macro_rules! register_uint_gauge { static INTERNAL_REGISTRY: Lazy = Lazy::new(Registry::new); /// Register a collector in the internal registry. MUST be called before the first call to `gather()`. +/// /// Otherwise, we can have a deadlock in the `gather()` call, trying to register a new collector /// while holding the lock. pub fn register_internal(c: Box) -> prometheus::Result<()> { diff --git a/libs/pageserver_api/src/controller_api.rs b/libs/pageserver_api/src/controller_api.rs index 94104af002..5c8dcbf571 100644 --- a/libs/pageserver_api/src/controller_api.rs +++ b/libs/pageserver_api/src/controller_api.rs @@ -147,6 +147,8 @@ pub struct TenantDescribeResponseShard { pub preferred_az_id: Option, } +/// Migration request for a given tenant shard to a given node. +/// /// Explicitly migrating a particular shard is a low level operation /// TODO: higher level "Reschedule tenant" operation where the request /// specifies some constraints, e.g. asking it to get off particular node(s) diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index d13d04eb1b..ffe79c8350 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -305,8 +305,10 @@ pub struct TenantConfig { pub lsn_lease_length_for_ts: Option, } -/// The policy for the aux file storage. It can be switched through `switch_aux_file_policy` -/// tenant config. When the first aux file written, the policy will be persisted in the +/// The policy for the aux file storage. +/// +/// It can be switched through `switch_aux_file_policy` tenant config. +/// When the first aux file written, the policy will be persisted in the /// `index_part.json` file and has a limited migration path. /// /// Currently, we only allow the following migration path: @@ -896,7 +898,9 @@ pub struct WalRedoManagerStatus { pub process: Option, } -/// The progress of a secondary tenant is mostly useful when doing a long running download: e.g. initiating +/// The progress of a secondary tenant. +/// +/// It is mostly useful when doing a long running download: e.g. initiating /// a download job, timing out while waiting for it to run, and then inspecting this status to understand /// what's happening. #[derive(Default, Debug, Serialize, Deserialize, Clone)] diff --git a/libs/postgres_backend/src/lib.rs b/libs/postgres_backend/src/lib.rs index 7c7c6535b3..600f1d728c 100644 --- a/libs/postgres_backend/src/lib.rs +++ b/libs/postgres_backend/src/lib.rs @@ -69,8 +69,10 @@ impl QueryError { } /// Returns true if the given error is a normal consequence of a network issue, -/// or the client closing the connection. These errors can happen during normal -/// operations, and don't indicate a bug in our code. +/// or the client closing the connection. +/// +/// These errors can happen during normal operations, +/// and don't indicate a bug in our code. pub fn is_expected_io_error(e: &io::Error) -> bool { use io::ErrorKind::*; matches!( diff --git a/libs/postgres_connection/src/lib.rs b/libs/postgres_connection/src/lib.rs index 9f57f3d507..ddf9f7b610 100644 --- a/libs/postgres_connection/src/lib.rs +++ b/libs/postgres_connection/src/lib.rs @@ -7,6 +7,7 @@ use std::fmt; use url::Host; /// Parses a string of format either `host:port` or `host` into a corresponding pair. +/// /// The `host` part should be a correct `url::Host`, while `port` (if present) should be /// a valid decimal u16 of digits only. pub fn parse_host_port>(host_port: S) -> Result<(Host, Option), anyhow::Error> { diff --git a/libs/remote_storage/src/lib.rs b/libs/remote_storage/src/lib.rs index cc1d3e0ae4..b5b69c9faf 100644 --- a/libs/remote_storage/src/lib.rs +++ b/libs/remote_storage/src/lib.rs @@ -45,6 +45,8 @@ pub use azure_core::Etag; pub use error::{DownloadError, TimeTravelError, TimeoutOrCancel}; +/// Default concurrency limit for S3 operations +/// /// Currently, sync happens with AWS S3, that has two limits on requests per second: /// ~200 RPS for IAM services /// @@ -300,7 +302,9 @@ pub trait RemoteStorage: Send + Sync + 'static { ) -> Result<(), TimeTravelError>; } -/// DownloadStream is sensitive to the timeout and cancellation used with the original +/// Data part of an ongoing [`Download`]. +/// +/// `DownloadStream` is sensitive to the timeout and cancellation used with the original /// [`RemoteStorage::download`] request. The type yields `std::io::Result` to be compatible /// with `tokio::io::copy_buf`. // This has 'static because safekeepers do not use cancellation tokens (yet) diff --git a/libs/tenant_size_model/src/lib.rs b/libs/tenant_size_model/src/lib.rs index a3e12cf0e3..974a498404 100644 --- a/libs/tenant_size_model/src/lib.rs +++ b/libs/tenant_size_model/src/lib.rs @@ -5,9 +5,10 @@ mod calculation; pub mod svg; -/// StorageModel is the input to the synthetic size calculation. It represents -/// a tree of timelines, with just the information that's needed for the -/// calculation. This doesn't track timeline names or where each timeline +/// StorageModel is the input to the synthetic size calculation. +/// +/// It represents a tree of timelines, with just the information that's needed +/// for the calculation. This doesn't track timeline names or where each timeline /// begins and ends, for example. Instead, it consists of "points of interest" /// on the timelines. A point of interest could be the timeline start or end point, /// the oldest point on a timeline that needs to be retained because of PITR diff --git a/libs/utils/src/circuit_breaker.rs b/libs/utils/src/circuit_breaker.rs index 720ea39d4f..e1ddfd8650 100644 --- a/libs/utils/src/circuit_breaker.rs +++ b/libs/utils/src/circuit_breaker.rs @@ -5,8 +5,10 @@ use std::{ use metrics::IntCounter; -/// Circuit breakers are for operations that are expensive and fallible: if they fail repeatedly, -/// we will stop attempting them for some period of time, to avoid denial-of-service from retries, and +/// Circuit breakers are for operations that are expensive and fallible. +/// +/// If a circuit breaker fails repeatedly, we will stop attempting it for some +/// period of time, to avoid denial-of-service from retries, and /// to mitigate the log spam from repeated failures. pub struct CircuitBreaker { /// An identifier that enables us to log useful errors when a circuit is broken diff --git a/libs/utils/src/id.rs b/libs/utils/src/id.rs index db468e3054..2cda899b15 100644 --- a/libs/utils/src/id.rs +++ b/libs/utils/src/id.rs @@ -249,8 +249,10 @@ macro_rules! id_newtype { }; } -/// Neon timeline IDs are different from PostgreSQL timeline -/// IDs. They serve a similar purpose though: they differentiate +/// Neon timeline ID. +/// +/// They are different from PostgreSQL timeline +/// IDs, but serve a similar purpose: they differentiate /// between different "histories" of the same cluster. However, /// PostgreSQL timeline IDs are a bit cumbersome, because they are only /// 32-bits wide, and they must be in ascending order in any given diff --git a/libs/utils/src/lock_file.rs b/libs/utils/src/lock_file.rs index 59c66ca757..3a2ed3e830 100644 --- a/libs/utils/src/lock_file.rs +++ b/libs/utils/src/lock_file.rs @@ -100,7 +100,9 @@ pub enum LockFileRead { } /// Open & try to lock the lock file at the given `path`, returning a [handle][`LockFileRead`] to -/// inspect its content. It is not an `Err(...)` if the file does not exist or is already locked. +/// inspect its content. +/// +/// It is not an `Err(...)` if the file does not exist or is already locked. /// Check the [`LockFileRead`] variants for details. pub fn read_and_hold_lock_file(path: &Utf8Path) -> anyhow::Result { let res = fs::OpenOptions::new().read(true).open(path); diff --git a/libs/utils/src/pageserver_feedback.rs b/libs/utils/src/pageserver_feedback.rs index 3ddfa44f41..dede65e699 100644 --- a/libs/utils/src/pageserver_feedback.rs +++ b/libs/utils/src/pageserver_feedback.rs @@ -8,6 +8,7 @@ use tracing::{trace, warn}; use crate::lsn::Lsn; /// Feedback pageserver sends to safekeeper and safekeeper resends to compute. +/// /// Serialized in custom flexible key/value format. In replication protocol, it /// is marked with NEON_STATUS_UPDATE_TAG_BYTE to differentiate from postgres /// Standby status update / Hot standby feedback messages. diff --git a/libs/utils/src/poison.rs b/libs/utils/src/poison.rs index 27378c69fc..c3e2fba20c 100644 --- a/libs/utils/src/poison.rs +++ b/libs/utils/src/poison.rs @@ -65,6 +65,8 @@ impl Poison { } } +/// Armed pointer to a [`Poison`]. +/// /// Use [`Self::data`] and [`Self::data_mut`] to access the wrapped state. /// Once modifications are done, use [`Self::disarm`]. /// If [`Guard`] gets dropped instead of calling [`Self::disarm`], the state is poisoned diff --git a/libs/utils/src/shard.rs b/libs/utils/src/shard.rs index f6b430657e..d146010b41 100644 --- a/libs/utils/src/shard.rs +++ b/libs/utils/src/shard.rs @@ -13,10 +13,11 @@ pub struct ShardNumber(pub u8); #[derive(Ord, PartialOrd, Eq, PartialEq, Clone, Copy, Serialize, Deserialize, Debug, Hash)] pub struct ShardCount(pub u8); -/// Combination of ShardNumber and ShardCount. For use within the context of a particular tenant, -/// when we need to know which shard we're dealing with, but do not need to know the full -/// ShardIdentity (because we won't be doing any page->shard mapping), and do not need to know -/// the fully qualified TenantShardId. +/// Combination of ShardNumber and ShardCount. +/// +/// For use within the context of a particular tenant, when we need to know which shard we're +/// dealing with, but do not need to know the full ShardIdentity (because we won't be doing +/// any page->shard mapping), and do not need to know the fully qualified TenantShardId. #[derive(Eq, PartialEq, PartialOrd, Ord, Clone, Copy, Hash)] pub struct ShardIndex { pub shard_number: ShardNumber, diff --git a/libs/utils/src/simple_rcu.rs b/libs/utils/src/simple_rcu.rs index ecc5353be3..01750b2aef 100644 --- a/libs/utils/src/simple_rcu.rs +++ b/libs/utils/src/simple_rcu.rs @@ -49,12 +49,11 @@ use std::sync::{RwLock, RwLockWriteGuard}; use tokio::sync::watch; -/// /// Rcu allows multiple readers to read and hold onto a value without blocking -/// (for very long). Storing to the Rcu updates the value, making new readers -/// immediately see the new value, but it also waits for all current readers to -/// finish. +/// (for very long). /// +/// Storing to the Rcu updates the value, making new readers immediately see +/// the new value, but it also waits for all current readers to finish. pub struct Rcu { inner: RwLock>, } diff --git a/libs/utils/src/sync/heavier_once_cell.rs b/libs/utils/src/sync/heavier_once_cell.rs index 1abd3d9861..dc711fb028 100644 --- a/libs/utils/src/sync/heavier_once_cell.rs +++ b/libs/utils/src/sync/heavier_once_cell.rs @@ -5,7 +5,9 @@ use std::sync::{ use tokio::sync::Semaphore; /// Custom design like [`tokio::sync::OnceCell`] but using [`OwnedSemaphorePermit`] instead of -/// `SemaphorePermit`, allowing use of `take` which does not require holding an outer mutex guard +/// `SemaphorePermit`. +/// +/// Allows use of `take` which does not require holding an outer mutex guard /// for the duration of initialization. /// /// Has no unsafe, builds upon [`tokio::sync::Semaphore`] and [`std::sync::Mutex`]. diff --git a/libs/utils/src/vec_map.rs b/libs/utils/src/vec_map.rs index 18b2af14f1..5f0028bacd 100644 --- a/libs/utils/src/vec_map.rs +++ b/libs/utils/src/vec_map.rs @@ -7,6 +7,7 @@ pub enum VecMapOrdering { } /// Ordered map datastructure implemented in a Vec. +/// /// Append only - can only add keys that are larger than the /// current max key. /// Ordering can be adjusted using [`VecMapOrdering`] diff --git a/libs/utils/src/yielding_loop.rs b/libs/utils/src/yielding_loop.rs index 41c4cee45d..68274f0631 100644 --- a/libs/utils/src/yielding_loop.rs +++ b/libs/utils/src/yielding_loop.rs @@ -6,9 +6,10 @@ pub enum YieldingLoopError { Cancelled, } -/// Helper for long synchronous loops, e.g. over all tenants in the system. Periodically -/// yields to avoid blocking the executor, and after resuming checks the provided -/// cancellation token to drop out promptly on shutdown. +/// Helper for long synchronous loops, e.g. over all tenants in the system. +/// +/// Periodically yields to avoid blocking the executor, and after resuming +/// checks the provided cancellation token to drop out promptly on shutdown. #[inline(always)] pub async fn yielding_loop( interval: usize, diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index 4e68e276d3..29a98855d3 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -180,6 +180,8 @@ pub struct PageServerConf { pub io_buffer_alignment: usize, } +/// Token for authentication to safekeepers +/// /// We do not want to store this in a PageServerConf because the latter may be logged /// and/or serialized at a whim, while the token is secret. Currently this token is the /// same for accessing all tenants/timelines, but may become per-tenant/per-timeline in diff --git a/pageserver/src/context.rs b/pageserver/src/context.rs index 012cb8d96f..7afcf52cf2 100644 --- a/pageserver/src/context.rs +++ b/pageserver/src/context.rs @@ -1,7 +1,9 @@ -//! This module defines `RequestContext`, a structure that we use throughout -//! the pageserver to propagate high-level context from places -//! that _originate_ activity down to the shared code paths at the -//! heart of the pageserver. It's inspired by Golang's `context.Context`. +//! Defines [`RequestContext`]. +//! +//! It is a structure that we use throughout the pageserver to propagate +//! high-level context from places that _originate_ activity down to the +//! shared code paths at the heart of the pageserver. It's inspired by +//! Golang's `context.Context`. //! //! For example, in `Timeline::get(page_nr, lsn)` we need to answer the following questions: //! 1. What high-level activity ([`TaskKind`]) needs this page? diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index d28a214265..808d4b666e 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -1021,9 +1021,10 @@ impl Timeline { } /// DatadirModification represents an operation to ingest an atomic set of -/// updates to the repository. It is created by the 'begin_record' -/// function. It is called for each WAL record, so that all the modifications -/// by a one WAL record appear atomic. +/// updates to the repository. +/// +/// It is created by the 'begin_record' function. It is called for each WAL +/// record, so that all the modifications by a one WAL record appear atomic. pub struct DatadirModification<'a> { /// The timeline this modification applies to. You can access this to /// read the state, but note that any pending updates are *not* reflected @@ -2048,6 +2049,7 @@ impl<'a> DatadirModification<'a> { /// This struct facilitates accessing either a committed key from the timeline at a /// specific LSN, or the latest uncommitted key from a pending modification. +/// /// During WAL ingestion, the records from multiple LSNs may be batched in the same /// modification before being flushed to the timeline. Hence, the routines in WalIngest /// need to look up the keys in the modification first before looking them up in the diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index fb30857ddf..fd2520a42e 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -1,8 +1,9 @@ +//! Timeline repository implementation that keeps old data in layer files, and +//! the recent changes in ephemeral files. //! -//! Timeline repository implementation that keeps old data in files on disk, and -//! the recent changes in memory. See tenant/*_layer.rs files. -//! The functions here are responsible for locating the correct layer for the -//! get/put call, walking back the timeline branching history as needed. +//! See tenant/*_layer.rs files. The functions here are responsible for locating +//! the correct layer for the get/put call, walking back the timeline branching +//! history as needed. //! //! The files are stored in the .neon/tenants//timelines/ //! directory. See docs/pageserver-storage.md for how the files are managed. diff --git a/pageserver/src/tenant/metadata.rs b/pageserver/src/tenant/metadata.rs index 190316df42..24440d4b35 100644 --- a/pageserver/src/tenant/metadata.rs +++ b/pageserver/src/tenant/metadata.rs @@ -1,7 +1,8 @@ -//! Describes the legacy now hopefully no longer modified per-timeline metadata stored in -//! `index_part.json` managed by [`remote_timeline_client`]. For many tenants and their timelines, -//! this struct and it's original serialization format is still needed because they were written a -//! long time ago. +//! Describes the legacy now hopefully no longer modified per-timeline metadata. +//! +//! It is stored in `index_part.json` managed by [`remote_timeline_client`]. For many tenants and +//! their timelines, this struct and its original serialization format is still needed because +//! they were written a long time ago. //! //! Instead of changing and adding versioning to this, just change [`IndexPart`] with soft json //! versioning. diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 4e6ea0c8f9..2104f41531 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -282,9 +282,10 @@ impl BackgroundPurges { static TENANTS: Lazy> = Lazy::new(|| std::sync::RwLock::new(TenantsMap::Initializing)); -/// The TenantManager is responsible for storing and mutating the collection of all tenants -/// that this pageserver process has state for. Every Tenant and SecondaryTenant instance -/// lives inside the TenantManager. +/// Responsible for storing and mutating the collection of all tenants +/// that this pageserver has state for. +/// +/// Every Tenant and SecondaryTenant instance lives inside the TenantManager. /// /// The most important role of the TenantManager is to prevent conflicts: e.g. trying to attach /// the same tenant twice concurrently, or trying to configure the same tenant into secondary @@ -2346,8 +2347,9 @@ pub enum TenantMapError { ShuttingDown, } -/// Guards a particular tenant_id's content in the TenantsMap. While this -/// structure exists, the TenantsMap will contain a [`TenantSlot::InProgress`] +/// Guards a particular tenant_id's content in the TenantsMap. +/// +/// While this structure exists, the TenantsMap will contain a [`TenantSlot::InProgress`] /// for this tenant, which acts as a marker for any operations targeting /// this tenant to retry later, or wait for the InProgress state to end. /// diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 71b766e4c7..1f9ae40af5 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -2184,6 +2184,8 @@ pub fn remote_timeline_path( remote_timelines_path(tenant_shard_id).join(Utf8Path::new(&timeline_id.to_string())) } +/// Obtains the path of the given Layer in the remote +/// /// Note that the shard component of a remote layer path is _not_ always the same /// as in the TenantShardId of the caller: tenants may reference layers from a different /// ShardIndex. Use the ShardIndex from the layer's metadata. diff --git a/pageserver/src/tenant/remote_timeline_client/index.rs b/pageserver/src/tenant/remote_timeline_client/index.rs index 757fb9d032..c51ff54919 100644 --- a/pageserver/src/tenant/remote_timeline_client/index.rs +++ b/pageserver/src/tenant/remote_timeline_client/index.rs @@ -1,4 +1,5 @@ //! In-memory index to track the tenant files on the remote storage. +//! //! Able to restore itself from the storage index parts, that are located in every timeline's remote directory and contain all data about //! remote timeline layers and its metadata. diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index a1202ad507..dac6b2f893 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -434,10 +434,11 @@ impl ReadableLayer { } } -/// Layers contain a hint indicating whether they are likely to be used for reads. This is a hint rather -/// than an authoritative value, so that we do not have to update it synchronously when changing the visibility -/// of layers (for example when creating a branch that makes some previously covered layers visible). It should -/// be used for cache management but not for correctness-critical checks. +/// Layers contain a hint indicating whether they are likely to be used for reads. +/// +/// This is a hint rather than an authoritative value, so that we do not have to update it synchronously +/// when changing the visibility of layers (for example when creating a branch that makes some previously +/// covered layers visible). It should be used for cache management but not for correctness-critical checks. #[derive(Debug, Clone, PartialEq, Eq)] pub enum LayerVisibilityHint { /// A Visible layer might be read while serving a read, because there is not an image layer between it diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 6a2cd94232..34f1b15138 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -136,10 +136,11 @@ impl Summary { // Flag indicating that this version initialize the page const WILL_INIT: u64 = 1; -/// Struct representing reference to BLOB in layers. Reference contains BLOB -/// offset, and for WAL records it also contains `will_init` flag. The flag -/// helps to determine the range of records that needs to be applied, without -/// reading/deserializing records themselves. +/// Struct representing reference to BLOB in layers. +/// +/// Reference contains BLOB offset, and for WAL records it also contains +/// `will_init` flag. The flag helps to determine the range of records +/// that needs to be applied, without reading/deserializing records themselves. #[derive(Debug, Serialize, Deserialize, Copy, Clone)] pub struct BlobRef(pub u64); diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index 77ce1ae670..875e223c9c 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -1,7 +1,9 @@ //! An ImageLayer represents an image or a snapshot of a key-range at -//! one particular LSN. It contains an image of all key-value pairs -//! in its key-range. Any key that falls into the image layer's range -//! but does not exist in the layer, does not exist. +//! one particular LSN. +//! +//! It contains an image of all key-value pairs in its key-range. Any key +//! that falls into the image layer's range but does not exist in the layer, +//! does not exist. //! //! An image layer is stored in a file on disk. The file is stored in //! timelines/ directory. Currently, there are no diff --git a/pageserver/src/tenant/storage_layer/layer_desc.rs b/pageserver/src/tenant/storage_layer/layer_desc.rs index cbd18e650f..e90ff3c4b2 100644 --- a/pageserver/src/tenant/storage_layer/layer_desc.rs +++ b/pageserver/src/tenant/storage_layer/layer_desc.rs @@ -12,8 +12,10 @@ use serde::{Deserialize, Serialize}; #[cfg(test)] use utils::id::TenantId; -/// A unique identifier of a persistent layer. This is different from `LayerDescriptor`, which is only used in the -/// benchmarks. This struct contains all necessary information to find the image / delta layer. It also provides +/// A unique identifier of a persistent layer. +/// +/// This is different from `LayerDescriptor`, which is only used in the benchmarks. +/// This struct contains all necessary information to find the image / delta layer. It also provides /// a unified way to generate layer information like file name. #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Hash)] pub struct PersistentLayerDesc { diff --git a/pageserver/src/tenant/storage_layer/layer_name.rs b/pageserver/src/tenant/storage_layer/layer_name.rs index 47ae556279..ffe7ca5f3e 100644 --- a/pageserver/src/tenant/storage_layer/layer_name.rs +++ b/pageserver/src/tenant/storage_layer/layer_name.rs @@ -217,8 +217,9 @@ impl fmt::Display for ImageLayerName { } } -/// LayerName is the logical identity of a layer within a LayerMap at a moment in time. The -/// LayerName is not a unique filename, as the same LayerName may have multiple physical incarnations +/// LayerName is the logical identity of a layer within a LayerMap at a moment in time. +/// +/// The LayerName is not a unique filename, as the same LayerName may have multiple physical incarnations /// over time (e.g. across shard splits or compression). The physical filenames of layers in local /// storage and object names in remote storage consist of the LayerName plus some extra qualifiers /// that uniquely identify the physical incarnation of a layer (see [crate::tenant::remote_timeline_client::remote_layer_path]) diff --git a/pageserver/src/tenant/storage_layer/merge_iterator.rs b/pageserver/src/tenant/storage_layer/merge_iterator.rs index d2c341e5ce..0831fd9530 100644 --- a/pageserver/src/tenant/storage_layer/merge_iterator.rs +++ b/pageserver/src/tenant/storage_layer/merge_iterator.rs @@ -226,9 +226,11 @@ impl<'a> IteratorWrapper<'a> { } } -/// A merge iterator over delta/image layer iterators. When duplicated records are -/// found, the iterator will not perform any deduplication, and the caller should handle -/// these situation. By saying duplicated records, there are many possibilities: +/// A merge iterator over delta/image layer iterators. +/// +/// When duplicated records are found, the iterator will not perform any +/// deduplication, and the caller should handle these situation. By saying +/// duplicated records, there are many possibilities: /// /// * Two same delta at the same LSN. /// * Two same image at the same LSN. diff --git a/pageserver/src/tenant/storage_layer/split_writer.rs b/pageserver/src/tenant/storage_layer/split_writer.rs index e8deb0a1e5..7c1ac863bf 100644 --- a/pageserver/src/tenant/storage_layer/split_writer.rs +++ b/pageserver/src/tenant/storage_layer/split_writer.rs @@ -34,9 +34,10 @@ impl SplitWriterResult { } } -/// An image writer that takes images and produces multiple image layers. The interface does not -/// guarantee atomicity (i.e., if the image layer generation fails, there might be leftover files -/// to be cleaned up) +/// An image writer that takes images and produces multiple image layers. +/// +/// The interface does not guarantee atomicity (i.e., if the image layer generation +/// fails, there might be leftover files to be cleaned up) #[must_use] pub struct SplitImageLayerWriter { inner: ImageLayerWriter, @@ -193,9 +194,10 @@ impl SplitImageLayerWriter { } } -/// A delta writer that takes key-lsn-values and produces multiple delta layers. The interface does not -/// guarantee atomicity (i.e., if the delta layer generation fails, there might be leftover files -/// to be cleaned up). +/// A delta writer that takes key-lsn-values and produces multiple delta layers. +/// +/// The interface does not guarantee atomicity (i.e., if the delta layer generation fails, +/// there might be leftover files to be cleaned up). /// /// Note that if updates of a single key exceed the target size limit, all of the updates will be batched /// into a single file. This behavior might change in the future. For reference, the legacy compaction algorithm diff --git a/pageserver/src/tenant/vectored_blob_io.rs b/pageserver/src/tenant/vectored_blob_io.rs index 4d51dc442d..553edf6d8b 100644 --- a/pageserver/src/tenant/vectored_blob_io.rs +++ b/pageserver/src/tenant/vectored_blob_io.rs @@ -593,8 +593,10 @@ impl<'a> VectoredBlobReader<'a> { } } -/// Read planner used in [`crate::tenant::storage_layer::image_layer::ImageLayerIterator`]. It provides a streaming API for -/// getting read blobs. It returns a batch when `handle` gets called and when the current key would just exceed the read_size and +/// Read planner used in [`crate::tenant::storage_layer::image_layer::ImageLayerIterator`]. +/// +/// It provides a streaming API for getting read blobs. It returns a batch when +/// `handle` gets called and when the current key would just exceed the read_size and /// max_cnt constraints. pub struct StreamingVectoredReadPlanner { read_builder: Option, diff --git a/pageserver/src/virtual_file.rs b/pageserver/src/virtual_file.rs index ed6ff86c10..57856eea80 100644 --- a/pageserver/src/virtual_file.rs +++ b/pageserver/src/virtual_file.rs @@ -1,6 +1,7 @@ -//! //! VirtualFile is like a normal File, but it's not bound directly to -//! a file descriptor. Instead, the file is opened when it's read from, +//! a file descriptor. +//! +//! Instead, the file is opened when it's read from, //! and if too many files are open globally in the system, least-recently //! used ones are closed. //! diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index 82585f9ed8..a36955fa21 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -43,13 +43,12 @@ use utils::lsn::Lsn; use utils::sync::gate::GateError; use utils::sync::heavier_once_cell; +/// The real implementation that uses a Postgres process to +/// perform WAL replay. /// -/// This is the real implementation that uses a Postgres process to -/// perform WAL replay. Only one thread can use the process at a time, -/// that is controlled by the Mutex. In the future, we might want to -/// launch a pool of processes to allow concurrent replay of multiple -/// records. -/// +/// Only one thread can use the process at a time, that is controlled by the +/// Mutex. In the future, we might want to launch a pool of processes to allow +/// concurrent replay of multiple records. pub struct PostgresRedoManager { tenant_shard_id: TenantShardId, conf: &'static PageServerConf, diff --git a/proxy/src/stream.rs b/proxy/src/stream.rs index 332dc27787..c14dd18afe 100644 --- a/proxy/src/stream.rs +++ b/proxy/src/stream.rs @@ -14,6 +14,7 @@ use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; use tokio_rustls::server::TlsStream; /// Stream wrapper which implements libpq's protocol. +/// /// NOTE: This object deliberately doesn't implement [`AsyncRead`] /// or [`AsyncWrite`] to prevent subtle errors (e.g. trying /// to pass random malformed bytes through the connection). diff --git a/safekeeper/src/pull_timeline.rs b/safekeeper/src/pull_timeline.rs index 600a6bd8f0..64585f5edc 100644 --- a/safekeeper/src/pull_timeline.rs +++ b/safekeeper/src/pull_timeline.rs @@ -484,6 +484,7 @@ pub async fn validate_temp_timeline( } /// Move timeline from a temp directory to the main storage, and load it to the global map. +/// /// This operation is done under a lock to prevent bugs if several concurrent requests are /// trying to load the same timeline. Note that it doesn't guard against creating the /// timeline with the same ttid, but no one should be doing this anyway. diff --git a/safekeeper/src/receive_wal.rs b/safekeeper/src/receive_wal.rs index ab8c76dc17..e35f806e90 100644 --- a/safekeeper/src/receive_wal.rs +++ b/safekeeper/src/receive_wal.rs @@ -448,8 +448,10 @@ async fn network_write( const KEEPALIVE_INTERVAL: Duration = Duration::from_secs(1); /// Encapsulates a task which takes messages from msg_rx, processes and pushes -/// replies to reply_tx; reading from socket and writing to disk in parallel is -/// beneficial for performance, this struct provides writing to disk part. +/// replies to reply_tx. +/// +/// Reading from socket and writing to disk in parallel is beneficial for +/// performance, this struct provides the writing to disk part. pub struct WalAcceptor { tli: WalResidentTimeline, msg_rx: Receiver, diff --git a/safekeeper/src/state.rs b/safekeeper/src/state.rs index dca6414082..97eeae3638 100644 --- a/safekeeper/src/state.rs +++ b/safekeeper/src/state.rs @@ -147,9 +147,11 @@ pub struct TimelineMemState { pub proposer_uuid: PgUuid, } -/// Safekeeper persistent state plus in memory layer, to avoid frequent fsyncs -/// when we update fields like commit_lsn which don't need immediate -/// persistence. Provides transactional like API to atomically update the state. +/// Safekeeper persistent state plus in memory layer. +/// +/// Allows us to avoid frequent fsyncs when we update fields like commit_lsn +/// which don't need immediate persistence. Provides transactional like API +/// to atomically update the state. /// /// Implements Deref into *persistent* part. pub struct TimelineState { diff --git a/safekeeper/src/timeline.rs b/safekeeper/src/timeline.rs index 95ee925e1a..6fd5de0ad6 100644 --- a/safekeeper/src/timeline.rs +++ b/safekeeper/src/timeline.rs @@ -169,6 +169,7 @@ impl<'a> Drop for WriteGuardSharedState<'a> { } /// This structure is stored in shared state and represents the state of the timeline. +/// /// Usually it holds SafeKeeper, but it also supports offloaded timeline state. In this /// case, SafeKeeper is not available (because WAL is not present on disk) and all /// operations can be done only with control file. diff --git a/safekeeper/src/timeline_eviction.rs b/safekeeper/src/timeline_eviction.rs index 5d0567575c..5aa4921a92 100644 --- a/safekeeper/src/timeline_eviction.rs +++ b/safekeeper/src/timeline_eviction.rs @@ -1,6 +1,8 @@ -//! Code related to evicting WAL files to remote storage. The actual upload is done by the -//! partial WAL backup code. This file has code to delete and re-download WAL files, -//! cross-validate with partial WAL backup if local file is still present. +//! Code related to evicting WAL files to remote storage. +//! +//! The actual upload is done by the partial WAL backup code. This file has +//! code to delete and re-download WAL files, cross-validate with partial WAL +//! backup if local file is still present. use anyhow::Context; use camino::Utf8PathBuf; diff --git a/safekeeper/src/timeline_guard.rs b/safekeeper/src/timeline_guard.rs index dbdf46412d..1ddac573d2 100644 --- a/safekeeper/src/timeline_guard.rs +++ b/safekeeper/src/timeline_guard.rs @@ -1,4 +1,6 @@ -//! Timeline residence guard is needed to ensure that WAL segments are present on disk, +//! Timeline residence guard +//! +//! It is needed to ensure that WAL segments are present on disk, //! as long as the code is holding the guard. This file implements guard logic, to issue //! and drop guards, and to notify the manager when the guard is dropped. diff --git a/safekeeper/src/timeline_manager.rs b/safekeeper/src/timeline_manager.rs index f997f48454..6be75479db 100644 --- a/safekeeper/src/timeline_manager.rs +++ b/safekeeper/src/timeline_manager.rs @@ -1,4 +1,5 @@ //! The timeline manager task is responsible for managing the timeline's background tasks. +//! //! It is spawned alongside each timeline and exits when the timeline is deleted. //! It watches for changes in the timeline state and decides when to spawn or kill background tasks. //! It also can manage some reactive state, like should the timeline be active for broker pushes or not. diff --git a/safekeeper/src/timelines_set.rs b/safekeeper/src/timelines_set.rs index d6eea79f82..096e348295 100644 --- a/safekeeper/src/timelines_set.rs +++ b/safekeeper/src/timelines_set.rs @@ -60,7 +60,8 @@ impl TimelinesSet { } } -/// Guard is used to add or remove timeline from the set. +/// Guard is used to add or remove timelines from the set. +/// /// If the timeline present in set, it will be removed from it on drop. /// Note: do not use more than one guard for the same timeline, it caches the presence state. /// It is designed to be used in the manager task only. diff --git a/safekeeper/src/wal_backup_partial.rs b/safekeeper/src/wal_backup_partial.rs index 4050a82fff..bddfca50e4 100644 --- a/safekeeper/src/wal_backup_partial.rs +++ b/safekeeper/src/wal_backup_partial.rs @@ -1,6 +1,8 @@ //! Safekeeper timeline has a background task which is subscribed to `commit_lsn` -//! and `flush_lsn` updates. After the partial segment was updated (`flush_lsn` -//! was changed), the segment will be uploaded to S3 in about 15 minutes. +//! and `flush_lsn` updates. +//! +//! After the partial segment was updated (`flush_lsn` was changed), the segment +//! will be uploaded to S3 within the configured `partial_backup_timeout`. //! //! The filename format for partial segments is //! `Segment_Term_Flush_Commit_skNN.partial`, where: diff --git a/safekeeper/src/wal_service.rs b/safekeeper/src/wal_service.rs index 16f7748eb4..1ab54d4cce 100644 --- a/safekeeper/src/wal_service.rs +++ b/safekeeper/src/wal_service.rs @@ -17,6 +17,7 @@ use crate::SafeKeeperConf; use postgres_backend::{AuthType, PostgresBackend}; /// Accept incoming TCP connections and spawn them into a background thread. +/// /// allowed_auth_scope is either SafekeeperData (wide JWT tokens giving access /// to any tenant are allowed) or Tenant (only tokens giving access to specific /// tenant are allowed). Doesn't matter if auth is disabled in conf. diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 324f864291..e7eae647df 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -117,7 +117,9 @@ pub(crate) const STARTUP_RECONCILE_TIMEOUT: Duration = Duration::from_secs(30); pub const MAX_OFFLINE_INTERVAL_DEFAULT: Duration = Duration::from_secs(30); /// How long a node may be unresponsive to heartbeats during start up before we declare it -/// offline. This is much more lenient than [`MAX_OFFLINE_INTERVAL_DEFAULT`] since the pageserver's +/// offline. +/// +/// This is much more lenient than [`MAX_OFFLINE_INTERVAL_DEFAULT`] since the pageserver's /// handling of the re-attach response may take a long time and blocks heartbeats from /// being handled on the pageserver side. pub const MAX_WARMING_UP_INTERVAL_DEFAULT: Duration = Duration::from_secs(300); diff --git a/storage_scrubber/src/garbage.rs b/storage_scrubber/src/garbage.rs index 3e22960f8d..d53611ed6e 100644 --- a/storage_scrubber/src/garbage.rs +++ b/storage_scrubber/src/garbage.rs @@ -1,6 +1,7 @@ -//! Functionality for finding and purging garbage, as in "garbage collection". Garbage means -//! S3 objects which are either not referenced by any metadata, or are referenced by a -//! control plane tenant/timeline in a deleted state. +//! Functionality for finding and purging garbage, as in "garbage collection". +//! +//! Garbage means S3 objects which are either not referenced by any metadata, +//! or are referenced by a control plane tenant/timeline in a deleted state. use std::{ collections::{HashMap, HashSet}, diff --git a/storage_scrubber/src/metadata_stream.rs b/storage_scrubber/src/metadata_stream.rs index 10d77937f1..f896cff2d5 100644 --- a/storage_scrubber/src/metadata_stream.rs +++ b/storage_scrubber/src/metadata_stream.rs @@ -74,7 +74,9 @@ pub async fn stream_tenant_shards<'a>( } /// Given a `TenantShardId`, output a stream of the timelines within that tenant, discovered -/// using a listing. The listing is done before the stream is built, so that this +/// using a listing. +/// +/// The listing is done before the stream is built, so that this /// function can be used to generate concurrency on a stream using buffer_unordered. pub async fn stream_tenant_timelines<'a>( remote_client: &'a GenericRemoteStorage, diff --git a/storage_scrubber/src/pageserver_physical_gc.rs b/storage_scrubber/src/pageserver_physical_gc.rs index 88681e38c2..c96d9cad3b 100644 --- a/storage_scrubber/src/pageserver_physical_gc.rs +++ b/storage_scrubber/src/pageserver_physical_gc.rs @@ -440,9 +440,10 @@ async fn gc_ancestor( Ok(()) } -/// Physical garbage collection: removing unused S3 objects. This is distinct from the garbage collection -/// done inside the pageserver, which operates at a higher level (keys, layers). This type of garbage collection -/// is about removing: +/// Physical garbage collection: removing unused S3 objects. +/// +/// This is distinct from the garbage collection done inside the pageserver, which operates at a higher level +/// (keys, layers). This type of garbage collection is about removing: /// - Objects that were uploaded but never referenced in the remote index (e.g. because of a shutdown between /// uploading a layer and uploading an index) /// - Index objects from historic generations From e287f36a058221b7c804b4b0f440933962eb3deb Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Fri, 6 Sep 2024 15:23:57 +0300 Subject: [PATCH 35/45] safekeeper: fix endpoint restart immediately after xlog switch. Check that truncation point is not from the future by comparing it with write_record_lsn, not write_lsn, and explain that xlog switch changes their normal order. ref https://github.com/neondatabase/neon/issues/8911 --- safekeeper/src/safekeeper.rs | 3 ++- safekeeper/src/wal_storage.rs | 23 ++++++++++++++++++----- test_runner/regress/test_wal_acceptor.py | 18 ++++++++++++++++++ 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/safekeeper/src/safekeeper.rs b/safekeeper/src/safekeeper.rs index dbe0034de2..b3e006ab05 100644 --- a/safekeeper/src/safekeeper.rs +++ b/safekeeper/src/safekeeper.rs @@ -938,8 +938,9 @@ where } trace!( - "processed AppendRequest of len {}, end_lsn={:?}, commit_lsn={:?}, truncate_lsn={:?}, flushed={:?}", + "processed AppendRequest of len {}, begin_lsn={}, end_lsn={:?}, commit_lsn={:?}, truncate_lsn={:?}, flushed={:?}", msg.wal_data.len(), + msg.h.begin_lsn, msg.h.end_lsn, msg.h.commit_lsn, msg.h.truncate_lsn, diff --git a/safekeeper/src/wal_storage.rs b/safekeeper/src/wal_storage.rs index 89c2e98a94..c477fe5c7b 100644 --- a/safekeeper/src/wal_storage.rs +++ b/safekeeper/src/wal_storage.rs @@ -98,7 +98,19 @@ pub struct PhysicalStorage { /// Also can be ahead of record_lsn, if happen to be in the middle of a WAL record. write_lsn: Lsn, - /// The LSN of the last WAL record written to disk. Still can be not fully flushed. + /// The LSN of the last WAL record written to disk. Still can be not fully + /// flushed. + /// + /// Note: Normally it (and flush_record_lsn) is <= write_lsn, but after xlog + /// switch ingest the reverse is true because we don't bump write_lsn up to + /// the next segment: WAL stream from the compute doesn't have the gap and + /// for simplicity / as a sanity check we disallow any non-sequential + /// writes, so write zeros as is. + /// + /// Similar effect is in theory possible due to LSN alignment: if record + /// ends at *2, decoder will report end lsn as *8 even though we haven't + /// written these zeros yet. In practice compute likely never sends + /// non-aligned chunks of data. write_record_lsn: Lsn, /// The LSN of the last WAL record flushed to disk. @@ -440,11 +452,12 @@ impl Storage for PhysicalStorage { .with_label_values(&["truncate_wal"]) .start_timer(); - // Streaming must not create a hole, so truncate cannot be called on non-written lsn - if self.write_lsn != Lsn(0) && end_pos > self.write_lsn { + // Streaming must not create a hole, so truncate cannot be called on + // non-written lsn. + if self.write_record_lsn != Lsn(0) && end_pos > self.write_record_lsn { bail!( - "truncate_wal called on non-written WAL, write_lsn={}, end_pos={}", - self.write_lsn, + "truncate_wal called on non-written WAL, write_record_lsn={}, end_pos={}", + self.write_record_lsn, end_pos ); } diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index 3785651aed..5672e836ee 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -1057,6 +1057,24 @@ def test_restart_endpoint(neon_env_builder: NeonEnvBuilder): endpoint.start() +# Try restarting endpoint immediately after xlog switch. +# https://github.com/neondatabase/neon/issues/8911 +def test_restart_endpoint_after_switch_wal(neon_env_builder: NeonEnvBuilder): + env = neon_env_builder.init_start() + + endpoint = env.endpoints.create_start("main") + + endpoint.safe_psql("create table t (i int)") + + endpoint.safe_psql("SELECT pg_switch_wal()") + + # we want immediate shutdown to have endpoint restart on xlog switch record, + # so prevent shutdown checkpoint. + endpoint.stop(mode="immediate") + endpoint = env.endpoints.create_start("main") + endpoint.safe_psql("SELECT 'works'") + + # Context manager which logs passed time on exit. class DurationLogger: def __init__(self, desc): From af6f63617e7421fca62ad2bf7ebfe2f0de66a793 Mon Sep 17 00:00:00 2001 From: Folke Behrens Date: Fri, 6 Sep 2024 17:13:30 +0200 Subject: [PATCH 36/45] proxy: clean up code and lints for 1.81 and 1.82 (#8945) --- proxy/src/cache/timed_lru.rs | 2 +- proxy/src/lib.rs | 25 +++++++++++++++---------- proxy/src/scram/exchange.rs | 2 ++ proxy/src/serverless/sql_over_http.rs | 9 +++------ 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/proxy/src/cache/timed_lru.rs b/proxy/src/cache/timed_lru.rs index 8bb482f7c6..5b08d74696 100644 --- a/proxy/src/cache/timed_lru.rs +++ b/proxy/src/cache/timed_lru.rs @@ -16,7 +16,7 @@ use tracing::debug; // On the other hand, `hashlink` has good download stats and appears to be maintained. use hashlink::{linked_hash_map::RawEntryMut, LruCache}; -use super::{common::Cached, *}; +use super::{common::Cached, timed_lru, Cache}; /// An implementation of timed LRU cache with fixed capacity. /// Key properties: diff --git a/proxy/src/lib.rs b/proxy/src/lib.rs index 8d7e586b3d..923d6ae288 100644 --- a/proxy/src/lib.rs +++ b/proxy/src/lib.rs @@ -44,16 +44,14 @@ clippy::items_after_statements, )] // List of temporarily allowed lints. -// TODO: Switch to except() once stable with 1.81. // TODO: fix code and reduce list or move to permanent list above. -#![allow( +#![expect( clippy::cargo_common_metadata, clippy::cast_possible_truncation, clippy::cast_possible_wrap, clippy::cast_precision_loss, clippy::cast_sign_loss, clippy::doc_markdown, - clippy::implicit_hasher, clippy::inline_always, clippy::match_same_arms, clippy::match_wild_err_arm, @@ -61,21 +59,28 @@ clippy::missing_panics_doc, clippy::module_name_repetitions, clippy::needless_pass_by_value, - clippy::needless_raw_string_hashes, clippy::redundant_closure_for_method_calls, - clippy::return_self_not_must_use, clippy::similar_names, clippy::single_match_else, clippy::struct_excessive_bools, clippy::struct_field_names, clippy::too_many_lines, - clippy::unreadable_literal, - clippy::unused_async, - clippy::unused_self, - clippy::wildcard_imports + clippy::unused_self +)] +#![cfg_attr( + any(test, feature = "testing"), + allow( + clippy::needless_raw_string_hashes, + clippy::unreadable_literal, + clippy::unused_async, + ) )] // List of temporarily allowed lints to unblock beta/nightly. -#![allow(unknown_lints, clippy::manual_inspect)] +#![allow( + unknown_lints, + // TODO: 1.82: Add `use` where necessary and remove from this list. + impl_trait_overcaptures, +)] use std::{convert::Infallible, future::Future}; diff --git a/proxy/src/scram/exchange.rs b/proxy/src/scram/exchange.rs index 786cbcaa19..afb5604666 100644 --- a/proxy/src/scram/exchange.rs +++ b/proxy/src/scram/exchange.rs @@ -217,6 +217,7 @@ impl sasl::Mechanism for Exchange<'_> { self.state = ExchangeState::SaltSent(sent); Ok(Step::Continue(self, msg)) } + #[allow(unreachable_patterns)] // TODO: 1.82: simply drop this match Step::Success(x, _) => match x {}, Step::Failure(msg) => Ok(Step::Failure(msg)), } @@ -224,6 +225,7 @@ impl sasl::Mechanism for Exchange<'_> { ExchangeState::SaltSent(sent) => { match sent.transition(self.secret, &self.tls_server_end_point, input)? { Step::Success(keys, msg) => Ok(Step::Success(keys, msg)), + #[allow(unreachable_patterns)] // TODO: 1.82: simply drop this match Step::Continue(x, _) => match x {}, Step::Failure(msg) => Ok(Step::Failure(msg)), } diff --git a/proxy/src/serverless/sql_over_http.rs b/proxy/src/serverless/sql_over_http.rs index 5b36f5e91d..2188edc8c5 100644 --- a/proxy/src/serverless/sql_over_http.rs +++ b/proxy/src/serverless/sql_over_http.rs @@ -745,22 +745,20 @@ impl BatchQueryData { builder = builder.deferrable(true); } - let transaction = builder.start().await.map_err(|e| { + let transaction = builder.start().await.inspect_err(|_| { // if we cannot start a transaction, we should return immediately // and not return to the pool. connection is clearly broken discard.discard(); - e })?; let json_output = match query_batch(cancel.child_token(), &transaction, self, parsed_headers).await { Ok(json_output) => { info!("commit"); - let status = transaction.commit().await.map_err(|e| { + let status = transaction.commit().await.inspect_err(|_| { // if we cannot commit - for now don't return connection to pool // TODO: get a query status from the error discard.discard(); - e })?; discard.check_idle(status); json_output @@ -776,11 +774,10 @@ impl BatchQueryData { } Err(err) => { info!("rollback"); - let status = transaction.rollback().await.map_err(|e| { + let status = transaction.rollback().await.inspect_err(|_| { // if we cannot rollback - for now don't return connection to pool // TODO: get a query status from the error discard.discard(); - e })?; discard.check_idle(status); return Err(err); From 11cf16e3f363ce027e53b1834a77858d50daee0d Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Mon, 19 Aug 2024 14:42:07 +0300 Subject: [PATCH 37/45] safekeeper: add term_bump endpoint. When walproposer observes now higher term it restarts instead of crashing whole compute with PANIC; this avoids compute crash after term_bump call. After successfull election we're still checking last_log_term of the highest given vote to ensure basebackup is good, and PANIC otherwise. It will be used for migration per 035-safekeeper-dynamic-membership-change.md and https://github.com/neondatabase/docs/pull/21 ref https://github.com/neondatabase/neon/issues/8700 --- libs/safekeeper_api/src/models.rs | 13 +++++++++ pgxn/neon/walproposer.c | 24 ++++++++++----- safekeeper/src/auth.rs | 3 ++ safekeeper/src/http/routes.rs | 28 +++++++++++++++++- safekeeper/src/state.rs | 26 +++++++++++++++-- safekeeper/src/timeline.rs | 10 +++++++ test_runner/fixtures/safekeeper/http.py | 29 +++++++++++++++++++ test_runner/regress/test_wal_acceptor.py | 37 ++++++++++++++++++++++++ 8 files changed, 159 insertions(+), 11 deletions(-) diff --git a/libs/safekeeper_api/src/models.rs b/libs/safekeeper_api/src/models.rs index 2fbc333075..28666d197a 100644 --- a/libs/safekeeper_api/src/models.rs +++ b/libs/safekeeper_api/src/models.rs @@ -60,3 +60,16 @@ pub struct TimelineCopyRequest { pub target_timeline_id: TimelineId, pub until_lsn: Lsn, } + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct TimelineTermBumpRequest { + /// bump to + pub term: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct TimelineTermBumpResponse { + // before the request + pub previous_term: u64, + pub current_term: u64, +} diff --git a/pgxn/neon/walproposer.c b/pgxn/neon/walproposer.c index c53257923a..c1914421ec 100644 --- a/pgxn/neon/walproposer.c +++ b/pgxn/neon/walproposer.c @@ -1038,9 +1038,12 @@ DetermineEpochStartLsn(WalProposer *wp) if (SkipXLogPageHeader(wp, wp->propEpochStartLsn) != wp->api.get_redo_start_lsn(wp)) { /* - * However, allow to proceed if previously elected leader was me; - * plain restart of walproposer not intervened by concurrent - * compute (who could generate WAL) is ok. + * However, allow to proceed if last_log_term on the node which gave + * the highest vote (i.e. point where we are going to start writing) + * actually had been won by me; plain restart of walproposer not + * intervened by concurrent compute which wrote WAL is ok. + * + * This avoids compute crash after manual term_bump. */ if (!((dth->n_entries >= 1) && (dth->entries[dth->n_entries - 1].term == pg_atomic_read_u64(&walprop_shared->mineLastElectedTerm)))) @@ -1442,12 +1445,17 @@ RecvAppendResponses(Safekeeper *sk) if (sk->appendResponse.term > wp->propTerm) { /* - * Another compute with higher term is running. Panic to restart - * PG as we likely need to retake basebackup. However, don't dump - * core as this is kinda expected scenario. + * + * Term has changed to higher one, probably another compute is + * running. If this is the case we could PANIC as well because + * likely it inserted some data and our basebackup is unsuitable + * anymore. However, we also bump term manually (term_bump endpoint) + * on safekeepers for migration purposes, in this case we do want + * compute to stay alive. So restart walproposer with FATAL instead + * of panicking; if basebackup is spoiled next election will notice + * this. */ - disable_core_dump(); - wp_log(PANIC, "WAL acceptor %s:%s with term " INT64_FORMAT " rejected our request, our term " INT64_FORMAT ", meaning another compute is running at the same time, and it conflicts with us", + wp_log(FATAL, "WAL acceptor %s:%s with term " INT64_FORMAT " rejected our request, our term " INT64_FORMAT ", meaning another compute is running at the same time, and it conflicts with us", sk->host, sk->port, sk->appendResponse.term, wp->propTerm); } diff --git a/safekeeper/src/auth.rs b/safekeeper/src/auth.rs index b8bc3f3e06..c5c9393c00 100644 --- a/safekeeper/src/auth.rs +++ b/safekeeper/src/auth.rs @@ -1,6 +1,9 @@ use utils::auth::{AuthError, Claims, Scope}; use utils::id::TenantId; +/// If tenant_id is provided, allow if token (claims) is for this tenant or +/// whole safekeeper scope (SafekeeperData). Else, allow only if token is +/// SafekeeperData. pub fn check_permission(claims: &Claims, tenant_id: Option) -> Result<(), AuthError> { match (&claims.scope, tenant_id) { (Scope::Tenant, None) => Err(AuthError( diff --git a/safekeeper/src/http/routes.rs b/safekeeper/src/http/routes.rs index 9b7424a818..e482edea55 100644 --- a/safekeeper/src/http/routes.rs +++ b/safekeeper/src/http/routes.rs @@ -18,8 +18,8 @@ use utils::http::endpoint::{prometheus_metrics_handler, request_span, ChannelWri use utils::http::request::parse_query_param; use postgres_ffi::WAL_SEGMENT_SIZE; -use safekeeper_api::models::TimelineCreateRequest; use safekeeper_api::models::{SkTimelineInfo, TimelineCopyRequest}; +use safekeeper_api::models::{TimelineCreateRequest, TimelineTermBumpRequest}; use utils::{ auth::SwappableJwtAuth, http::{ @@ -408,6 +408,28 @@ async fn timeline_backup_partial_reset(request: Request) -> Result, +) -> Result, ApiError> { + let ttid = TenantTimelineId::new( + parse_request_param(&request, "tenant_id")?, + parse_request_param(&request, "timeline_id")?, + ); + check_permission(&request, Some(ttid.tenant_id))?; + + let request_data: TimelineTermBumpRequest = json_request(&mut request).await?; + + let tli = GlobalTimelines::get(ttid).map_err(ApiError::from)?; + let response = tli + .term_bump(request_data.term) + .await + .map_err(ApiError::InternalServerError)?; + + json_response(StatusCode::OK, response) +} + /// Used only in tests to hand craft required data. async fn record_safekeeper_info(mut request: Request) -> Result, ApiError> { let ttid = TenantTimelineId::new( @@ -630,6 +652,10 @@ pub fn make_router(conf: SafeKeeperConf) -> RouterBuilder "/v1/tenant/:tenant_id/timeline/:timeline_id/backup_partial_reset", |r| request_span(r, timeline_backup_partial_reset), ) + .post( + "/v1/tenant/:tenant_id/timeline/:timeline_id/term_bump", + |r| request_span(r, timeline_term_bump_handler), + ) .post("/v1/record_safekeeper_info/:tenant_id/:timeline_id", |r| { request_span(r, record_safekeeper_info) }) diff --git a/safekeeper/src/state.rs b/safekeeper/src/state.rs index 97eeae3638..8ae749ded5 100644 --- a/safekeeper/src/state.rs +++ b/safekeeper/src/state.rs @@ -1,9 +1,10 @@ //! Defines per timeline data stored persistently (SafeKeeperPersistentState) //! and its wrapper with in memory layer (SafekeeperState). -use std::ops::Deref; +use std::{cmp::max, ops::Deref}; use anyhow::Result; +use safekeeper_api::models::TimelineTermBumpResponse; use serde::{Deserialize, Serialize}; use utils::{ id::{NodeId, TenantId, TenantTimelineId, TimelineId}, @@ -12,7 +13,7 @@ use utils::{ use crate::{ control_file, - safekeeper::{AcceptorState, PersistedPeerInfo, PgUuid, ServerInfo, TermHistory}, + safekeeper::{AcceptorState, PersistedPeerInfo, PgUuid, ServerInfo, Term, TermHistory}, wal_backup_partial::{self}, }; @@ -211,6 +212,27 @@ where let s = self.start_change(); self.finish_change(&s).await } + + /// Make term at least as `to`. If `to` is None, increment current one. This + /// is not in safekeeper.rs because we want to be able to do it even if + /// timeline is offloaded. + pub async fn term_bump(&mut self, to: Option) -> Result { + let before = self.acceptor_state.term; + let mut state = self.start_change(); + let new = match to { + Some(to) => max(state.acceptor_state.term, to), + None => state.acceptor_state.term + 1, + }; + if new > state.acceptor_state.term { + state.acceptor_state.term = new; + self.finish_change(&state).await?; + } + let after = self.acceptor_state.term; + Ok(TimelineTermBumpResponse { + previous_term: before, + current_term: after, + }) + } } impl Deref for TimelineState diff --git a/safekeeper/src/timeline.rs b/safekeeper/src/timeline.rs index 6fd5de0ad6..fb98534768 100644 --- a/safekeeper/src/timeline.rs +++ b/safekeeper/src/timeline.rs @@ -4,6 +4,7 @@ use anyhow::{anyhow, bail, Result}; use camino::Utf8PathBuf; use remote_storage::RemotePath; +use safekeeper_api::models::TimelineTermBumpResponse; use serde::{Deserialize, Serialize}; use tokio::fs::{self}; use tokio_util::sync::CancellationToken; @@ -215,6 +216,10 @@ impl StateSK { .get_last_log_term(self.flush_lsn()) } + pub async fn term_bump(&mut self, to: Option) -> Result { + self.state_mut().term_bump(to).await + } + /// Close open WAL files to release FDs. fn close_wal_store(&mut self) { if let StateSK::Loaded(sk) = self { @@ -854,6 +859,11 @@ impl Timeline { Ok(res) } + pub async fn term_bump(self: &Arc, to: Option) -> Result { + let mut state = self.write_shared_state().await; + state.sk.term_bump(to).await + } + /// Get the timeline guard for reading/writing WAL files. /// If WAL files are not present on disk (evicted), they will be automatically /// downloaded from remote storage. This is done in the manager task, which is diff --git a/test_runner/fixtures/safekeeper/http.py b/test_runner/fixtures/safekeeper/http.py index 9bf03554e7..96c84d1616 100644 --- a/test_runner/fixtures/safekeeper/http.py +++ b/test_runner/fixtures/safekeeper/http.py @@ -50,6 +50,19 @@ class SafekeeperMetrics(Metrics): ).value +@dataclass +class TermBumpResponse: + previous_term: int + current_term: int + + @classmethod + def from_json(cls, d: Dict[str, Any]) -> "TermBumpResponse": + return TermBumpResponse( + previous_term=d["previous_term"], + current_term=d["current_term"], + ) + + class SafekeeperHttpClient(requests.Session, MetricsGetter): HTTPError = requests.HTTPError @@ -252,6 +265,22 @@ class SafekeeperHttpClient(requests.Session, MetricsGetter): res.raise_for_status() return res.json() + def term_bump( + self, + tenant_id: TenantId, + timeline_id: TimelineId, + term: Optional[int], + ) -> TermBumpResponse: + body = {} + if term is not None: + body["term"] = term + res = self.post( + f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/term_bump", + json=body, + ) + res.raise_for_status() + return TermBumpResponse.from_json(res.json()) + def record_safekeeper_info(self, tenant_id: TenantId, timeline_id: TimelineId, body): res = self.post( f"http://localhost:{self.port}/v1/record_safekeeper_info/{tenant_id}/{timeline_id}", diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index 5672e836ee..50fac441c0 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -2194,6 +2194,43 @@ def test_patch_control_file(neon_env_builder: NeonEnvBuilder): assert res["timelines"][0]["control_file"]["timeline_start_lsn"] == "0/1" +def test_term_bump(neon_env_builder: NeonEnvBuilder): + neon_env_builder.num_safekeepers = 1 + env = neon_env_builder.init_start() + + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline + + endpoint = env.endpoints.create_start("main") + # initialize safekeeper + endpoint.safe_psql("create table t(key int, value text)") + + http_cli = env.safekeepers[0].http_client() + + # check that bump up to specific term works + curr_term = http_cli.timeline_status(tenant_id, timeline_id).term + bump_to = curr_term + 3 + res = http_cli.term_bump(tenant_id, timeline_id, bump_to) + log.info(f"bump to {bump_to} res: {res}") + assert res.current_term >= bump_to + + # check that bump to none increments current term + res = http_cli.term_bump(tenant_id, timeline_id, None) + log.info(f"bump to None res: {res}") + assert res.current_term > bump_to + assert res.current_term > res.previous_term + + # check that bumping doesn't work downward + res = http_cli.term_bump(tenant_id, timeline_id, 2) + log.info(f"bump to 2 res: {res}") + assert res.current_term > bump_to + assert res.current_term == res.previous_term + + # check that this doesn't kill endpoint because last WAL flush was his and + # thus its basebackup is still good + endpoint.safe_psql("insert into t values (1, 'payload')") + + # Test disables periodic pushes from safekeeper to the broker and checks that # pageserver can still discover safekeepers with discovery requests. def test_broker_discovery(neon_env_builder: NeonEnvBuilder): From 8eab7009c11ebc03f09b2f3916e642664a4b9f88 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Fri, 6 Sep 2024 16:54:45 +0300 Subject: [PATCH 38/45] safekeeper: do pid file lock before id init --- safekeeper/src/bin/safekeeper.rs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index 41c2d3fe08..644d5e6eaf 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -261,6 +261,15 @@ async fn main() -> anyhow::Result<()> { // Change into the data directory. std::env::set_current_dir(&workdir)?; + // Prevent running multiple safekeepers on the same directory + let lock_file_path = workdir.join(PID_FILE_NAME); + let lock_file = + pid_file::claim_for_current_process(&lock_file_path).context("claim pid file")?; + info!("claimed pid file at {lock_file_path:?}"); + // ensure that the lock file is held even if the main thread of the process is panics + // we need to release the lock file only when the current process is gone + std::mem::forget(lock_file); + // Set or read our ID. let id = set_id(&workdir, args.id.map(NodeId))?; if args.init { @@ -364,16 +373,6 @@ async fn main() -> anyhow::Result<()> { type JoinTaskRes = Result, JoinError>; async fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { - // Prevent running multiple safekeepers on the same directory - let lock_file_path = conf.workdir.join(PID_FILE_NAME); - let lock_file = - pid_file::claim_for_current_process(&lock_file_path).context("claim pid file")?; - info!("claimed pid file at {lock_file_path:?}"); - - // ensure that the lock file is held even if the main thread of the process is panics - // we need to release the lock file only when the current process is gone - std::mem::forget(lock_file); - info!("starting safekeeper WAL service on {}", conf.listen_pg_addr); let pg_listener = tcp_listener::bind(conf.listen_pg_addr.clone()).map_err(|e| { error!("failed to bind to address {}: {}", conf.listen_pg_addr, e); From c1a51416dbfadbf05cc352168ebc6fc4a83c6f59 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Fri, 6 Sep 2024 17:40:21 +0300 Subject: [PATCH 39/45] safekeeper: fsync filesystem on start. We can't really rely on files contents after boot without fsync'ing them. --- libs/utils/src/crashsafe.rs | 21 +++++++++++++++++++++ pageserver/src/bin/pageserver.rs | 19 ++----------------- safekeeper/src/bin/safekeeper.rs | 12 +++++++++++- safekeeper/src/wal_storage.rs | 3 +-- 4 files changed, 35 insertions(+), 20 deletions(-) diff --git a/libs/utils/src/crashsafe.rs b/libs/utils/src/crashsafe.rs index 756b19138c..946fedf6e5 100644 --- a/libs/utils/src/crashsafe.rs +++ b/libs/utils/src/crashsafe.rs @@ -1,9 +1,11 @@ +use std::os::fd::AsRawFd; use std::{ borrow::Cow, fs::{self, File}, io::{self, Write}, }; +use anyhow::Context; use camino::{Utf8Path, Utf8PathBuf}; /// Similar to [`std::fs::create_dir`], except we fsync the @@ -203,6 +205,25 @@ pub fn overwrite( Ok(()) } +/// Syncs the filesystem for the given file descriptor. +pub fn syncfs(fd: impl AsRawFd) -> anyhow::Result<()> { + // Linux guarantees durability for syncfs. + // POSIX doesn't have syncfs, and further does not actually guarantee durability of sync(). + #[cfg(target_os = "linux")] + { + nix::unistd::syncfs(fd.as_raw_fd()).context("syncfs")?; + } + #[cfg(target_os = "macos")] + { + // macOS is not a production platform for Neon, don't even bother. + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + compile_error!("Unsupported OS"); + } + Ok(()) +} + #[cfg(test)] mod tests { diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 59194ab4bd..d15a0e47a4 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -37,6 +37,7 @@ use pageserver::{ virtual_file, }; use postgres_backend::AuthType; +use utils::crashsafe::syncfs; use utils::failpoint_support; use utils::logging::TracingErrorLayerEnablement; use utils::{ @@ -155,23 +156,7 @@ fn main() -> anyhow::Result<()> { }; let started = Instant::now(); - // Linux guarantees durability for syncfs. - // POSIX doesn't have syncfs, and further does not actually guarantee durability of sync(). - #[cfg(target_os = "linux")] - { - use std::os::fd::AsRawFd; - nix::unistd::syncfs(dirfd.as_raw_fd()).context("syncfs")?; - } - #[cfg(target_os = "macos")] - { - // macOS is not a production platform for Neon, don't even bother. - drop(dirfd); - } - #[cfg(not(any(target_os = "linux", target_os = "macos")))] - { - compile_error!("Unsupported OS"); - } - + syncfs(dirfd)?; let elapsed = started.elapsed(); info!( elapsed_ms = elapsed.as_millis(), diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index 644d5e6eaf..5270934f5e 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -19,7 +19,7 @@ use std::fs::{self, File}; use std::io::{ErrorKind, Write}; use std::str::FromStr; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; use storage_broker::Uri; use tracing::*; @@ -373,6 +373,16 @@ async fn main() -> anyhow::Result<()> { type JoinTaskRes = Result, JoinError>; async fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { + // fsync the datadir to make sure we have a consistent state on disk. + let dfd = File::open(&conf.workdir).context("open datadir for syncfs")?; + let started = Instant::now(); + utils::crashsafe::syncfs(dfd)?; + let elapsed = started.elapsed(); + info!( + elapsed_ms = elapsed.as_millis(), + "syncfs data directory done" + ); + info!("starting safekeeper WAL service on {}", conf.listen_pg_addr); let pg_listener = tcp_listener::bind(conf.listen_pg_addr.clone()).map_err(|e| { error!("failed to bind to address {}: {}", conf.listen_pg_addr, e); diff --git a/safekeeper/src/wal_storage.rs b/safekeeper/src/wal_storage.rs index c477fe5c7b..46c260901d 100644 --- a/safekeeper/src/wal_storage.rs +++ b/safekeeper/src/wal_storage.rs @@ -179,8 +179,7 @@ impl PhysicalStorage { ) }; - // TODO: do we really know that write_lsn is fully flushed to disk? - // If not, maybe it's better to call fsync() here to be sure? + // note: this assumes we fsync'ed whole datadir on start. let flush_lsn = write_lsn; debug!( From 30583cb6264653175d659d0fcb636a42e21c5877 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Fri, 6 Sep 2024 17:42:35 +0100 Subject: [PATCH 40/45] CI(label-for-external-users): add retry logic for unexpected errors (#8938) ## Problem One of the PRs opened by a `neondatabase` org member got labelled as `external` because the `gh api` call failed in the wrong way: ``` Get "https://api.github.com/orgs/neondatabase/members/": dial tcp 140.82.114.5:443: i/o timeout is-member=false ``` ## Summary of changes - Check that the error message is expected before labelling PRs - Retry `gh api` call for 10 times in case of unexpected error messages - Add `workflow_dispatch` trigger --- .../workflows/label-for-external-users.yml | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/.github/workflows/label-for-external-users.yml b/.github/workflows/label-for-external-users.yml index 585d118dfb..b7cbc06a73 100644 --- a/.github/workflows/label-for-external-users.yml +++ b/.github/workflows/label-for-external-users.yml @@ -7,6 +7,11 @@ on: pull_request_target: types: - opened + workflow_dispatch: + inputs: + github-actor: + description: 'GitHub username. If empty, the username of the current user will be used' + required: false # No permission for GITHUB_TOKEN by default; the **minimal required** set of permissions should be granted in each job. permissions: {} @@ -26,12 +31,31 @@ jobs: id: check-user env: GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }} + ACTOR: ${{ inputs.github-actor || github.actor }} run: | - if gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "/orgs/${GITHUB_REPOSITORY_OWNER}/members/${GITHUB_ACTOR}"; then - is_member=true - else - is_member=false - fi + expected_error="User does not exist or is not a member of the organization" + output_file=output.txt + + for i in $(seq 1 10); do + if gh api "/orgs/${GITHUB_REPOSITORY_OWNER}/members/${ACTOR}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" > ${output_file}; then + + is_member=true + break + elif grep -q "${expected_error}" ${output_file}; then + is_member=false + break + elif [ $i -eq 10 ]; then + title="Failed to get memmbership status for ${ACTOR}" + message="The latest GitHub API error message: '$(cat ${output_file})'" + echo "::error file=.github/workflows/label-for-external-users.yml,title=${title}::${message}" + + exit 1 + fi + + sleep 1 + done echo "is-member=${is_member}" | tee -a ${GITHUB_OUTPUT} From ac5815b5940c412a281c6bbab34809689a738da7 Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." <4198311+skyzh@users.noreply.github.com> Date: Sat, 7 Sep 2024 02:14:21 +0800 Subject: [PATCH 41/45] feat(storage-controller): add node shards api (#8896) For control-plane managed tenants, we have the page in the admin console that lists all tenants on a specific pageserver. But for storage-controller managed ones, we don't have that functionality for now. ## Summary of changes Adds an API that lists all shards on a given node (intention + observed) --------- Signed-off-by: Alex Chi Z --- control_plane/storcon_cli/src/main.rs | 45 +++++++++++++++-- libs/pageserver_api/src/controller_api.rs | 15 ++++++ storage_controller/src/http.rs | 18 +++++++ storage_controller/src/service.rs | 49 +++++++++++++++++-- test_runner/fixtures/common_types.py | 8 +++ test_runner/fixtures/neon_fixtures.py | 26 +++++++++- .../regress/test_storage_controller.py | 6 +++ 7 files changed, 157 insertions(+), 10 deletions(-) diff --git a/control_plane/storcon_cli/src/main.rs b/control_plane/storcon_cli/src/main.rs index 2a81a3d825..651fcda8db 100644 --- a/control_plane/storcon_cli/src/main.rs +++ b/control_plane/storcon_cli/src/main.rs @@ -4,8 +4,8 @@ use std::{str::FromStr, time::Duration}; use clap::{Parser, Subcommand}; use pageserver_api::{ controller_api::{ - NodeAvailabilityWrapper, NodeDescribeResponse, ShardSchedulingPolicy, TenantCreateRequest, - TenantDescribeResponse, TenantPolicyRequest, + NodeAvailabilityWrapper, NodeDescribeResponse, NodeShardResponse, ShardSchedulingPolicy, + TenantCreateRequest, TenantDescribeResponse, TenantPolicyRequest, }, models::{ EvictionPolicy, EvictionPolicyLayerAccessThreshold, LocationConfigSecondary, @@ -80,7 +80,10 @@ enum Command { /// List nodes known to the storage controller Nodes {}, /// List tenants known to the storage controller - Tenants {}, + Tenants { + /// If this field is set, it will list the tenants on a specific node + node_id: Option, + }, /// Create a new tenant in the storage controller, and by extension on pageservers. TenantCreate { #[arg(long)] @@ -403,7 +406,41 @@ async fn main() -> anyhow::Result<()> { ) .await?; } - Command::Tenants {} => { + Command::Tenants { + node_id: Some(node_id), + } => { + let describe_response = storcon_client + .dispatch::<(), NodeShardResponse>( + Method::GET, + format!("control/v1/node/{node_id}/shards"), + None, + ) + .await?; + let shards = describe_response.shards; + let mut table = comfy_table::Table::new(); + table.set_header([ + "Shard", + "Intended Primary/Secondary", + "Observed Primary/Secondary", + ]); + for shard in shards { + table.add_row([ + format!("{}", shard.tenant_shard_id), + match shard.is_intended_secondary { + None => "".to_string(), + Some(true) => "Secondary".to_string(), + Some(false) => "Primary".to_string(), + }, + match shard.is_observed_secondary { + None => "".to_string(), + Some(true) => "Secondary".to_string(), + Some(false) => "Primary".to_string(), + }, + ]); + } + println!("{table}"); + } + Command::Tenants { node_id: None } => { let mut resp = storcon_client .dispatch::<(), Vec>( Method::GET, diff --git a/libs/pageserver_api/src/controller_api.rs b/libs/pageserver_api/src/controller_api.rs index 5c8dcbf571..40b7dbbbc2 100644 --- a/libs/pageserver_api/src/controller_api.rs +++ b/libs/pageserver_api/src/controller_api.rs @@ -112,6 +112,21 @@ pub struct TenantDescribeResponse { pub config: TenantConfig, } +#[derive(Serialize, Deserialize, Debug)] +pub struct NodeShardResponse { + pub node_id: NodeId, + pub shards: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct NodeShard { + pub tenant_shard_id: TenantShardId, + /// Whether the shard is observed secondary on a specific node. True = yes, False = no, None = not on this node. + pub is_observed_secondary: Option, + /// Whether the shard is intended to be a secondary on a specific node. True = yes, False = no, None = not on this node. + pub is_intended_secondary: Option, +} + #[derive(Serialize, Deserialize)] pub struct NodeDescribeResponse { pub id: NodeId, diff --git a/storage_controller/src/http.rs b/storage_controller/src/http.rs index 5d4d0460be..96bdd5039d 100644 --- a/storage_controller/src/http.rs +++ b/storage_controller/src/http.rs @@ -539,6 +539,17 @@ async fn handle_node_status(req: Request) -> Result, ApiErr json_response(StatusCode::OK, node_status) } +async fn handle_node_shards(req: Request) -> Result, ApiError> { + check_permissions(&req, Scope::Admin)?; + + let state = get_state(&req); + let node_id: NodeId = parse_request_param(&req, "node_id")?; + + let node_status = state.service.get_node_shards(node_id).await?; + + json_response(StatusCode::OK, node_status) +} + async fn handle_get_leader(req: Request) -> Result, ApiError> { check_permissions(&req, Scope::Admin)?; @@ -1109,6 +1120,13 @@ pub fn make_router( .get("/control/v1/node/:node_id", |r| { named_request_span(r, handle_node_status, RequestName("control_v1_node_status")) }) + .get("/control/v1/node/:node_id/shards", |r| { + named_request_span( + r, + handle_node_shards, + RequestName("control_v1_node_describe"), + ) + }) .get("/control/v1/leader", |r| { named_request_span(r, handle_get_leader, RequestName("control_v1_get_leader")) }) diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index e7eae647df..44fdb474b4 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -41,11 +41,11 @@ use itertools::Itertools; use pageserver_api::{ controller_api::{ MetadataHealthRecord, MetadataHealthUpdateRequest, NodeAvailability, NodeRegisterRequest, - NodeSchedulingPolicy, PlacementPolicy, ShardSchedulingPolicy, ShardsPreferredAzsRequest, - ShardsPreferredAzsResponse, TenantCreateRequest, TenantCreateResponse, - TenantCreateResponseShard, TenantDescribeResponse, TenantDescribeResponseShard, - TenantLocateResponse, TenantPolicyRequest, TenantShardMigrateRequest, - TenantShardMigrateResponse, + NodeSchedulingPolicy, NodeShard, NodeShardResponse, PlacementPolicy, ShardSchedulingPolicy, + ShardsPreferredAzsRequest, ShardsPreferredAzsResponse, TenantCreateRequest, + TenantCreateResponse, TenantCreateResponseShard, TenantDescribeResponse, + TenantDescribeResponseShard, TenantLocateResponse, TenantPolicyRequest, + TenantShardMigrateRequest, TenantShardMigrateResponse, }, models::{ SecondaryProgress, TenantConfigRequest, TimelineArchivalConfigRequest, @@ -4924,6 +4924,45 @@ impl Service { )) } + pub(crate) async fn get_node_shards( + &self, + node_id: NodeId, + ) -> Result { + let locked = self.inner.read().unwrap(); + let mut shards = Vec::new(); + for (tid, tenant) in locked.tenants.iter() { + let is_intended_secondary = match ( + tenant.intent.get_attached() == &Some(node_id), + tenant.intent.get_secondary().contains(&node_id), + ) { + (true, true) => { + return Err(ApiError::InternalServerError(anyhow::anyhow!( + "{} attached as primary+secondary on the same node", + tid + ))) + } + (true, false) => Some(false), + (false, true) => Some(true), + (false, false) => None, + }; + let is_observed_secondary = if let Some(ObservedStateLocation { conf: Some(conf) }) = + tenant.observed.locations.get(&node_id) + { + Some(conf.secondary_conf.is_some()) + } else { + None + }; + if is_intended_secondary.is_some() || is_observed_secondary.is_some() { + shards.push(NodeShard { + tenant_shard_id: *tid, + is_intended_secondary, + is_observed_secondary, + }); + } + } + Ok(NodeShardResponse { node_id, shards }) + } + pub(crate) async fn get_leader(&self) -> DatabaseResult> { self.persistence.get_leader().await } diff --git a/test_runner/fixtures/common_types.py b/test_runner/fixtures/common_types.py index 8eda19d1e2..064a678c96 100644 --- a/test_runner/fixtures/common_types.py +++ b/test_runner/fixtures/common_types.py @@ -140,6 +140,14 @@ class TenantId(Id): return self.id.hex() +class NodeId(Id): + def __repr__(self) -> str: + return f'`NodeId("{self.id.hex()}")' + + def __str__(self) -> str: + return self.id.hex() + + class TimelineId(Id): def __repr__(self) -> str: return f'TimelineId("{self.id.hex()}")' diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 18fbbde637..5a600dd0a1 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -62,7 +62,7 @@ from urllib3.util.retry import Retry from fixtures import overlayfs from fixtures.broker import NeonBroker -from fixtures.common_types import Lsn, TenantId, TenantShardId, TimelineId +from fixtures.common_types import Lsn, NodeId, TenantId, TenantShardId, TimelineId from fixtures.endpoint.http import EndpointHttpClient from fixtures.log_helper import log from fixtures.metrics import Metrics, MetricsGetter, parse_metrics @@ -2570,6 +2570,30 @@ class NeonStorageController(MetricsGetter, LogUtils): response.raise_for_status() return response.json() + def nodes(self): + """ + :return: list of {"id": ""} + """ + response = self.request( + "GET", + f"{self.api}/control/v1/node", + headers=self.headers(TokenScope.ADMIN), + ) + response.raise_for_status() + return response.json() + + def node_shards(self, node_id: NodeId): + """ + :return: list of {"shard_id": "", "is_secondary": bool} + """ + response = self.request( + "GET", + f"{self.api}/control/v1/node/{node_id}/shards", + headers=self.headers(TokenScope.ADMIN), + ) + response.raise_for_status() + return response.json() + def tenant_shard_split( self, tenant_id: TenantId, shard_count: int, shard_stripe_size: Optional[int] = None ) -> list[TenantShardId]: diff --git a/test_runner/regress/test_storage_controller.py b/test_runner/regress/test_storage_controller.py index 92cd74eba5..eea05d7548 100644 --- a/test_runner/regress/test_storage_controller.py +++ b/test_runner/regress/test_storage_controller.py @@ -1552,6 +1552,12 @@ def test_tenant_import(neon_env_builder: NeonEnvBuilder, shard_count, remote_sto literal_shard_count = 1 if shard_count is None else shard_count assert len(describe["shards"]) == literal_shard_count + nodes = env.storage_controller.nodes() + assert len(nodes) == 2 + describe1 = env.storage_controller.node_shards(nodes[0]["id"]) + describe2 = env.storage_controller.node_shards(nodes[1]["id"]) + assert len(describe1["shards"]) + len(describe2["shards"]) == literal_shard_count + # Check the data is still there: this implicitly proves that we recovered generation numbers # properly, for the timeline which was written to after a generation bump. for timeline, branch, expect_rows in [ From fa3fc73c1b3366a3316456bcb8fdce1bed159200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Fri, 6 Sep 2024 21:05:18 +0200 Subject: [PATCH 42/45] Address 1.82 clippy lints (#8944) Addresses the clippy lints of the beta 1.82 toolchain. The `too_long_first_doc_paragraph` lint complained a lot and was addressed separately: #8941 --- Dockerfile.build-tools | 2 +- libs/utils/src/logging.rs | 2 +- rust-toolchain.toml | 4 ++-- scripts/coverage | 4 ++-- storage_controller/src/service.rs | 10 +++++----- workspace_hack/Cargo.toml | 2 ++ 6 files changed, 13 insertions(+), 11 deletions(-) diff --git a/Dockerfile.build-tools b/Dockerfile.build-tools index a9cbed85fb..c4209c7a12 100644 --- a/Dockerfile.build-tools +++ b/Dockerfile.build-tools @@ -207,7 +207,7 @@ RUN curl -sSO https://static.rust-lang.org/rustup/dist/$(uname -m)-unknown-linux export PATH="$HOME/.cargo/bin:$PATH" && \ . "$HOME/.cargo/env" && \ cargo --version && rustup --version && \ - rustup component add llvm-tools-preview rustfmt clippy && \ + rustup component add llvm-tools rustfmt clippy && \ cargo install rustfilt --version ${RUSTFILT_VERSION} && \ cargo install cargo-hakari --version ${CARGO_HAKARI_VERSION} && \ cargo install cargo-deny --locked --version ${CARGO_DENY_VERSION} && \ diff --git a/libs/utils/src/logging.rs b/libs/utils/src/logging.rs index 71af43a4da..2ea0781667 100644 --- a/libs/utils/src/logging.rs +++ b/libs/utils/src/logging.rs @@ -190,7 +190,7 @@ impl Drop for TracingPanicHookGuard { } /// Named symbol for our panic hook, which logs the panic. -fn tracing_panic_hook(info: &std::panic::PanicInfo) { +fn tracing_panic_hook(info: &std::panic::PanicHookInfo) { // following rust 1.66.1 std implementation: // https://github.com/rust-lang/rust/blob/90743e7298aca107ddaa0c202a4d3604e29bfeb6/library/std/src/panicking.rs#L235-L288 let location = info.location(); diff --git a/rust-toolchain.toml b/rust-toolchain.toml index e78c4d6790..3c5d0b12a6 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -3,5 +3,5 @@ channel = "1.81.0" profile = "default" # The default profile includes rustc, rust-std, cargo, rust-docs, rustfmt and clippy. # https://rust-lang.github.io/rustup/concepts/profiles.html -# but we also need `llvm-tools-preview` for coverage data merges on CI -components = ["llvm-tools-preview", "rustfmt", "clippy"] +# but we also need `llvm-tools` for coverage data merges on CI +components = ["llvm-tools", "rustfmt", "clippy"] diff --git a/scripts/coverage b/scripts/coverage index 52a69c93b9..482dc58ff6 100755 --- a/scripts/coverage +++ b/scripts/coverage @@ -134,7 +134,7 @@ class LLVM: # Show a user-friendly warning raise Exception(' '.join([ f"It appears that you don't have `{name}` installed.", - "Please execute `rustup component add llvm-tools-preview`,", + "Please execute `rustup component add llvm-tools`,", "or install it via your package manager of choice.", "LLVM tools should be the same version as LLVM in `rustc --version --verbose`.", ])) @@ -518,7 +518,7 @@ def main() -> None: example = f""" prerequisites: # alternatively, install a system package for `llvm-tools` - rustup component add llvm-tools-preview + rustup component add llvm-tools self-contained example: {app} run make diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 44fdb474b4..6365423e10 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -451,7 +451,7 @@ struct ShardSplitParams { // When preparing for a shard split, we may either choose to proceed with the split, // or find that the work is already done and return NoOp. enum ShardSplitAction { - Split(ShardSplitParams), + Split(Box), NoOp(TenantShardSplitResponse), } @@ -4186,7 +4186,7 @@ impl Service { let policy = policy.unwrap(); let config = config.unwrap(); - Ok(ShardSplitAction::Split(ShardSplitParams { + Ok(ShardSplitAction::Split(Box::new(ShardSplitParams { old_shard_count, new_shard_count: ShardCount::new(split_req.new_shard_count), new_stripe_size: split_req.new_stripe_size, @@ -4194,13 +4194,13 @@ impl Service { policy, config, shard_ident, - })) + }))) } async fn do_tenant_shard_split( &self, tenant_id: TenantId, - params: ShardSplitParams, + params: Box, ) -> Result<(TenantShardSplitResponse, Vec), ApiError> { // FIXME: we have dropped self.inner lock, and not yet written anything to the database: another // request could occur here, deleting or mutating the tenant. begin_shard_split checks that the @@ -4216,7 +4216,7 @@ impl Service { policy, config, shard_ident, - } = params; + } = *params; // Drop any secondary locations: pageservers do not support splitting these, and in any case the // end-state for a split tenant will usually be to have secondary locations on different nodes. diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index 3d2fa8c214..94f4c0f22f 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -8,6 +8,8 @@ version = "0.1.0" description = "workspace-hack package, managed by hakari" # You can choose to publish this crate: see https://docs.rs/cargo-hakari/latest/cargo_hakari/publishing. publish = false +edition.workspace = true +license.workspace = true # The parts of the file between the BEGIN HAKARI SECTION and END HAKARI SECTION comments # are managed by hakari. From 3dbd34aa78258928344d4de80ddcdcf46b35dfbc Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Sat, 7 Sep 2024 00:42:55 +0300 Subject: [PATCH 43/45] feat(storcon): forward gc blocking and unblocking (#8956) Currently using gc blocking and unblocking with storage controller managed pageservers is painful. Implement the API on storage controller. Fixes: #8893 --- pageserver/client/src/lib.rs | 18 ++++ pageserver/client/src/mgmt_api.rs | 16 ++++ storage_controller/src/http.rs | 39 +++++++- storage_controller/src/pageserver_client.rs | 23 ++++- storage_controller/src/service.rs | 54 +++++++++++- .../regress/test_timeline_gc_blocking.py | 88 +++++++++++++++---- 6 files changed, 220 insertions(+), 18 deletions(-) diff --git a/pageserver/client/src/lib.rs b/pageserver/client/src/lib.rs index 4a3f4dea47..cc8db37173 100644 --- a/pageserver/client/src/lib.rs +++ b/pageserver/client/src/lib.rs @@ -1,2 +1,20 @@ pub mod mgmt_api; pub mod page_service; + +/// For timeline_block_unblock_gc, distinguish the two different operations. This could be a bool. +// If file structure is per-kind not per-feature then where to put this? +#[derive(Clone, Copy)] +pub enum BlockUnblock { + Block, + Unblock, +} + +impl std::fmt::Display for BlockUnblock { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + BlockUnblock::Block => "block", + BlockUnblock::Unblock => "unblock", + }; + f.write_str(s) + } +} diff --git a/pageserver/client/src/mgmt_api.rs b/pageserver/client/src/mgmt_api.rs index 737cb00835..a68f45a6d9 100644 --- a/pageserver/client/src/mgmt_api.rs +++ b/pageserver/client/src/mgmt_api.rs @@ -12,6 +12,8 @@ use utils::{ pub use reqwest::Body as ReqwestBody; +use crate::BlockUnblock; + pub mod util; #[derive(Debug, Clone)] @@ -454,6 +456,20 @@ impl Client { .map_err(Error::ReceiveBody) } + pub async fn timeline_block_unblock_gc( + &self, + tenant_shard_id: TenantShardId, + timeline_id: TimelineId, + dir: BlockUnblock, + ) -> Result<()> { + let uri = format!( + "{}/v1/tenant/{tenant_shard_id}/timeline/{timeline_id}/{dir}_gc", + self.mgmt_api_endpoint, + ); + + self.request(Method::POST, &uri, ()).await.map(|_| ()) + } + pub async fn tenant_reset(&self, tenant_shard_id: TenantShardId) -> Result<()> { let uri = format!( "{}/v1/tenant/{}/reset", diff --git a/storage_controller/src/http.rs b/storage_controller/src/http.rs index 96bdd5039d..a6638f5191 100644 --- a/storage_controller/src/http.rs +++ b/storage_controller/src/http.rs @@ -21,7 +21,7 @@ use pageserver_api::models::{ TenantTimeTravelRequest, TimelineArchivalConfigRequest, TimelineCreateRequest, }; use pageserver_api::shard::TenantShardId; -use pageserver_client::mgmt_api; +use pageserver_client::{mgmt_api, BlockUnblock}; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio_util::sync::CancellationToken; @@ -369,6 +369,23 @@ async fn handle_tenant_timeline_detach_ancestor( json_response(StatusCode::OK, res) } +async fn handle_tenant_timeline_block_unblock_gc( + service: Arc, + req: Request, + dir: BlockUnblock, +) -> Result, ApiError> { + let tenant_id: TenantId = parse_request_param(&req, "tenant_id")?; + check_permissions(&req, Scope::PageServerApi)?; + + let timeline_id: TimelineId = parse_request_param(&req, "timeline_id")?; + + service + .tenant_timeline_block_unblock_gc(tenant_id, timeline_id, dir) + .await?; + + json_response(StatusCode::OK, ()) +} + async fn handle_tenant_timeline_passthrough( service: Arc, req: Request, @@ -1292,6 +1309,26 @@ pub fn make_router( ) }, ) + .post( + "/v1/tenant/:tenant_id/timeline/:timeline_id/block_gc", + |r| { + tenant_service_handler( + r, + |s, r| handle_tenant_timeline_block_unblock_gc(s, r, BlockUnblock::Block), + RequestName("v1_tenant_timeline_block_unblock_gc"), + ) + }, + ) + .post( + "/v1/tenant/:tenant_id/timeline/:timeline_id/unblock_gc", + |r| { + tenant_service_handler( + r, + |s, r| handle_tenant_timeline_block_unblock_gc(s, r, BlockUnblock::Unblock), + RequestName("v1_tenant_timeline_block_unblock_gc"), + ) + }, + ) // Tenant detail GET passthrough to shard zero: .get("/v1/tenant/:tenant_id", |r| { tenant_service_handler( diff --git a/storage_controller/src/pageserver_client.rs b/storage_controller/src/pageserver_client.rs index 20770ed703..961a1f78dd 100644 --- a/storage_controller/src/pageserver_client.rs +++ b/storage_controller/src/pageserver_client.rs @@ -7,7 +7,10 @@ use pageserver_api::{ }, shard::TenantShardId, }; -use pageserver_client::mgmt_api::{Client, Result}; +use pageserver_client::{ + mgmt_api::{Client, Result}, + BlockUnblock, +}; use reqwest::StatusCode; use utils::id::{NodeId, TenantId, TimelineId}; @@ -258,6 +261,24 @@ impl PageserverClient { ) } + pub(crate) async fn timeline_block_unblock_gc( + &self, + tenant_shard_id: TenantShardId, + timeline_id: TimelineId, + dir: BlockUnblock, + ) -> Result<()> { + // measuring these makes no sense because we synchronize with the gc loop and remote + // storage on block_gc so there should be huge outliers + measured_request!( + "timeline_block_unblock_gc", + crate::metrics::Method::Post, + &self.node_id_label, + self.inner + .timeline_block_unblock_gc(tenant_shard_id, timeline_id, dir) + .await + ) + } + pub(crate) async fn get_utilization(&self) -> Result { measured_request!( "utilization", diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 6365423e10..be3efaf688 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -69,7 +69,7 @@ use pageserver_api::{ ValidateResponse, ValidateResponseTenant, }, }; -use pageserver_client::mgmt_api; +use pageserver_client::{mgmt_api, BlockUnblock}; use tokio::sync::mpsc::error::TrySendError; use tokio_util::sync::CancellationToken; use utils::{ @@ -142,6 +142,7 @@ enum TenantOperations { AttachHook, TimelineArchivalConfig, TimelineDetachAncestor, + TimelineGcBlockUnblock, } #[derive(Clone, strum_macros::Display)] @@ -3197,6 +3198,57 @@ impl Service { }).await? } + pub(crate) async fn tenant_timeline_block_unblock_gc( + &self, + tenant_id: TenantId, + timeline_id: TimelineId, + dir: BlockUnblock, + ) -> Result<(), ApiError> { + let _tenant_lock = trace_shared_lock( + &self.tenant_op_locks, + tenant_id, + TenantOperations::TimelineGcBlockUnblock, + ) + .await; + + self.tenant_remote_mutation(tenant_id, move |targets| async move { + if targets.is_empty() { + return Err(ApiError::NotFound( + anyhow::anyhow!("Tenant not found").into(), + )); + } + + async fn do_one( + tenant_shard_id: TenantShardId, + timeline_id: TimelineId, + node: Node, + jwt: Option, + dir: BlockUnblock, + ) -> Result<(), ApiError> { + let client = PageserverClient::new(node.get_id(), node.base_url(), jwt.as_deref()); + + client + .timeline_block_unblock_gc(tenant_shard_id, timeline_id, dir) + .await + .map_err(|e| passthrough_api_error(&node, e)) + } + + // no shard needs to go first/last; the operation should be idempotent + self.tenant_for_shards(targets, |tenant_shard_id, node| { + futures::FutureExt::boxed(do_one( + tenant_shard_id, + timeline_id, + node, + self.config.jwt_token.clone(), + dir, + )) + }) + .await + }) + .await??; + Ok(()) + } + /// Helper for concurrently calling a pageserver API on a number of shards, such as timeline creation. /// /// On success, the returned vector contains exactly the same number of elements as the input `locations`. diff --git a/test_runner/regress/test_timeline_gc_blocking.py b/test_runner/regress/test_timeline_gc_blocking.py index 24de894687..ddfe9b911f 100644 --- a/test_runner/regress/test_timeline_gc_blocking.py +++ b/test_runner/regress/test_timeline_gc_blocking.py @@ -1,17 +1,32 @@ import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from typing import List, Optional +import pytest +from fixtures.log_helper import log from fixtures.neon_fixtures import ( + LogCursor, NeonEnvBuilder, + NeonPageserver, ) from fixtures.pageserver.utils import wait_timeline_detail_404 -def test_gc_blocking_by_timeline(neon_env_builder: NeonEnvBuilder): +@pytest.mark.parametrize("sharded", [True, False]) +def test_gc_blocking_by_timeline(neon_env_builder: NeonEnvBuilder, sharded: bool): + neon_env_builder.num_pageservers = 2 if sharded else 1 env = neon_env_builder.init_start( - initial_tenant_conf={"gc_period": "1s", "lsn_lease_length": "0s"} + initial_tenant_conf={"gc_period": "1s", "lsn_lease_length": "0s"}, + initial_tenant_shard_count=2 if sharded else None, ) - ps = env.pageserver - http = ps.http_client() + + if sharded: + http = env.storage_controller.pageserver_api() + else: + http = env.pageserver.http_client() + + pss = ManyPageservers(list(map(lambda ps: ScrollableLog(ps, None), env.pageservers))) foo_branch = env.neon_cli.create_branch("foo", "main", env.initial_tenant) @@ -22,9 +37,8 @@ def test_gc_blocking_by_timeline(neon_env_builder: NeonEnvBuilder): tenant_before = http.tenant_status(env.initial_tenant) wait_for_another_gc_round() - _, offset = ps.assert_log_contains(gc_active_line) - - assert ps.log_contains(gc_skipped_line, offset) is None + pss.assert_log_contains(gc_active_line) + pss.assert_log_does_not_contain(gc_skipped_line) http.timeline_block_gc(env.initial_tenant, foo_branch) @@ -34,34 +48,78 @@ def test_gc_blocking_by_timeline(neon_env_builder: NeonEnvBuilder): assert gc_blocking == "BlockingReasons { timelines: 1, reasons: EnumSet(Manual) }" wait_for_another_gc_round() - _, offset = ps.assert_log_contains(gc_skipped_line, offset) + pss.assert_log_contains(gc_skipped_line) - ps.restart() - ps.quiesce_tenants() + pss.restart() + pss.quiesce_tenants() - _, offset = env.pageserver.assert_log_contains(init_gc_skipped, offset) + pss.assert_log_contains(init_gc_skipped) wait_for_another_gc_round() - _, offset = ps.assert_log_contains(gc_skipped_line, offset) + pss.assert_log_contains(gc_skipped_line) # deletion unblocks gc http.timeline_delete(env.initial_tenant, foo_branch) wait_timeline_detail_404(http, env.initial_tenant, foo_branch, 10, 1.0) wait_for_another_gc_round() - _, offset = ps.assert_log_contains(gc_active_line, offset) + pss.assert_log_contains(gc_active_line) http.timeline_block_gc(env.initial_tenant, env.initial_timeline) wait_for_another_gc_round() - _, offset = ps.assert_log_contains(gc_skipped_line, offset) + pss.assert_log_contains(gc_skipped_line) # removing the manual block also unblocks gc http.timeline_unblock_gc(env.initial_tenant, env.initial_timeline) wait_for_another_gc_round() - _, offset = ps.assert_log_contains(gc_active_line, offset) + pss.assert_log_contains(gc_active_line) def wait_for_another_gc_round(): time.sleep(2) + + +@dataclass +class ScrollableLog: + pageserver: NeonPageserver + offset: Optional[LogCursor] + + def assert_log_contains(self, what: str): + msg, offset = self.pageserver.assert_log_contains(what, offset=self.offset) + old = self.offset + self.offset = offset + log.info(f"{old} -> {offset}: {msg}") + + def assert_log_does_not_contain(self, what: str): + assert self.pageserver.log_contains(what) is None + + +@dataclass(frozen=True) +class ManyPageservers: + many: List[ScrollableLog] + + def assert_log_contains(self, what: str): + for one in self.many: + one.assert_log_contains(what) + + def assert_log_does_not_contain(self, what: str): + for one in self.many: + one.assert_log_does_not_contain(what) + + def restart(self): + def do_restart(x: ScrollableLog): + x.pageserver.restart() + + with ThreadPoolExecutor(max_workers=len(self.many)) as rt: + rt.map(do_restart, self.many) + rt.shutdown(wait=True) + + def quiesce_tenants(self): + def do_quiesce(x: ScrollableLog): + x.pageserver.quiesce_tenants() + + with ThreadPoolExecutor(max_workers=len(self.many)) as rt: + rt.map(do_quiesce, self.many) + rt.shutdown(wait=True) From 16c200d6d9f0eaade2efe4ad0f649c3bbf23bf08 Mon Sep 17 00:00:00 2001 From: Cihan Demirci <128653800+fcdm@users.noreply.github.com> Date: Sat, 7 Sep 2024 00:20:36 +0100 Subject: [PATCH 44/45] push images to prod ACR (#8940) Used `vars` for new storing non-sensitive information, changed dev secrets to vars as well but didn't cleanup any secrets. https://github.com/neondatabase/cloud/issues/16925 --------- Co-authored-by: Alexander Bayandin --- .github/actionlint.yml | 7 ++++ .github/workflows/_push-to-acr.yml | 56 ++++++++++++++++++++++++++++ .github/workflows/build_and_test.yml | 53 +++++++++++++------------- 3 files changed, 89 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/_push-to-acr.yml diff --git a/.github/actionlint.yml b/.github/actionlint.yml index 4ad8a7b460..1b602883c5 100644 --- a/.github/actionlint.yml +++ b/.github/actionlint.yml @@ -7,6 +7,13 @@ self-hosted-runner: - small-arm64 - us-east-2 config-variables: + - AZURE_DEV_CLIENT_ID + - AZURE_DEV_REGISTRY_NAME + - AZURE_DEV_SUBSCRIPTION_ID + - AZURE_PROD_CLIENT_ID + - AZURE_PROD_REGISTRY_NAME + - AZURE_PROD_SUBSCRIPTION_ID + - AZURE_TENANT_ID - BENCHMARK_PROJECT_ID_PUB - BENCHMARK_PROJECT_ID_SUB - REMOTE_STORAGE_AZURE_CONTAINER diff --git a/.github/workflows/_push-to-acr.yml b/.github/workflows/_push-to-acr.yml new file mode 100644 index 0000000000..415b3d9cc6 --- /dev/null +++ b/.github/workflows/_push-to-acr.yml @@ -0,0 +1,56 @@ +name: Push images to ACR +on: + workflow_call: + inputs: + client_id: + description: Client ID of Azure managed identity or Entra app + required: true + type: string + image_tag: + description: Tag for the container image + required: true + type: string + images: + description: Images to push + required: true + type: string + registry_name: + description: Name of the container registry + required: true + type: string + subscription_id: + description: Azure subscription ID + required: true + type: string + tenant_id: + description: Azure tenant ID + required: true + type: string + +jobs: + push-to-acr: + runs-on: ubuntu-22.04 + permissions: + contents: read # This is required for actions/checkout + id-token: write # This is required for Azure Login to work. + + steps: + - name: Azure login + uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # @v2.1.1 + with: + client-id: ${{ inputs.client_id }} + subscription-id: ${{ inputs.subscription_id }} + tenant-id: ${{ inputs.tenant_id }} + + - name: Login to ACR + run: | + az acr login --name=${{ inputs.registry_name }} + + - name: Copy docker images to ACR ${{ inputs.registry_name }} + run: | + images='${{ inputs.images }}' + for image in ${images}; do + docker buildx imagetools create \ + -t ${{ inputs.registry_name }}.azurecr.io/neondatabase/${image}:${{ inputs.image_tag }} \ + neondatabase/${image}:${{ inputs.image_tag }} + done diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index ee5fd1b0c6..4bb9e5cb66 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -794,9 +794,6 @@ jobs: docker compose -f ./docker-compose/docker-compose.yml down promote-images: - permissions: - contents: read # This is required for actions/checkout - id-token: write # This is required for Azure Login to work. needs: [ check-permissions, tag, test-images, vm-compute-node-image ] runs-on: ubuntu-22.04 @@ -823,28 +820,6 @@ jobs: neondatabase/vm-compute-node-${version}:${{ needs.tag.outputs.build-tag }} done - - name: Azure login - if: github.ref_name == 'main' - uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # @v2.1.1 - with: - client-id: ${{ secrets.AZURE_DEV_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_DEV_SUBSCRIPTION_ID }} - - - name: Login to ACR - if: github.ref_name == 'main' - run: | - az acr login --name=neoneastus2 - - - name: Copy docker images to ACR-dev - if: github.ref_name == 'main' - run: | - for image in neon compute-tools {vm-,}compute-node-{v14,v15,v16}; do - docker buildx imagetools create \ - -t neoneastus2.azurecr.io/neondatabase/${image}:${{ needs.tag.outputs.build-tag }} \ - neondatabase/${image}:${{ needs.tag.outputs.build-tag }} - done - - name: Add latest tag to images if: github.ref_name == 'main' run: | @@ -882,6 +857,30 @@ jobs: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/${image}:${{ needs.tag.outputs.build-tag }} done + push-to-acr-dev: + if: github.ref_name == 'main' + needs: [ tag, promote-images ] + uses: ./.github/workflows/_push-to-acr.yml + with: + client_id: ${{ vars.AZURE_DEV_CLIENT_ID }} + image_tag: ${{ needs.tag.outputs.build-tag }} + images: neon compute-tools vm-compute-node-v14 vm-compute-node-v15 vm-compute-node-v16 compute-node-v14 compute-node-v15 compute-node-v16 + registry_name: ${{ vars.AZURE_DEV_REGISTRY_NAME }} + subscription_id: ${{ vars.AZURE_DEV_SUBSCRIPTION_ID }} + tenant_id: ${{ vars.AZURE_TENANT_ID }} + + push-to-acr-prod: + if: github.ref_name == 'release'|| github.ref_name == 'release-proxy' + needs: [ tag, promote-images ] + uses: ./.github/workflows/_push-to-acr.yml + with: + client_id: ${{ vars.AZURE_PROD_CLIENT_ID }} + image_tag: ${{ needs.tag.outputs.build-tag }} + images: neon compute-tools vm-compute-node-v14 vm-compute-node-v15 vm-compute-node-v16 compute-node-v14 compute-node-v15 compute-node-v16 + registry_name: ${{ vars.AZURE_PROD_REGISTRY_NAME }} + subscription_id: ${{ vars.AZURE_PROD_SUBSCRIPTION_ID }} + tenant_id: ${{ vars.AZURE_TENANT_ID }} + trigger-custom-extensions-build-and-wait: needs: [ check-permissions, tag ] runs-on: ubuntu-22.04 @@ -957,8 +956,8 @@ jobs: exit 1 deploy: - needs: [ check-permissions, promote-images, tag, build-and-test-locally, trigger-custom-extensions-build-and-wait ] - if: github.ref_name == 'main' || github.ref_name == 'release'|| github.ref_name == 'release-proxy' + needs: [ check-permissions, promote-images, tag, build-and-test-locally, trigger-custom-extensions-build-and-wait, push-to-acr-dev, push-to-acr-prod ] + if: (github.ref_name == 'main' || github.ref_name == 'release' || github.ref_name == 'release-proxy') && !failure() && !cancelled() runs-on: [ self-hosted, small ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest From 7d7d1f354b127ae27ae2e76cb0b3e9a3c8f69d90 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Sat, 7 Sep 2024 08:17:25 +0100 Subject: [PATCH 45/45] Fix rust warnings on macOS (#8955) ## Problem ``` error: unused import: `anyhow::Context` --> libs/utils/src/crashsafe.rs:8:5 | 8 | use anyhow::Context; | ^^^^^^^^^^^^^^^ | = note: `-D unused-imports` implied by `-D warnings` = help: to override `-D warnings` add `#[allow(unused_imports)]` error: unused variable: `fd` --> libs/utils/src/crashsafe.rs:209:15 | 209 | pub fn syncfs(fd: impl AsRawFd) -> anyhow::Result<()> { | ^^ help: if this is intentional, prefix it with an underscore: `_fd` | = note: `-D unused-variables` implied by `-D warnings` = help: to override `-D warnings` add `#[allow(unused_variables)]` ``` ## Summary of changes - Fix rust warnings on macOS --- libs/utils/src/crashsafe.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/utils/src/crashsafe.rs b/libs/utils/src/crashsafe.rs index 946fedf6e5..b97c6c7a45 100644 --- a/libs/utils/src/crashsafe.rs +++ b/libs/utils/src/crashsafe.rs @@ -5,7 +5,6 @@ use std::{ io::{self, Write}, }; -use anyhow::Context; use camino::{Utf8Path, Utf8PathBuf}; /// Similar to [`std::fs::create_dir`], except we fsync the @@ -206,11 +205,13 @@ pub fn overwrite( } /// Syncs the filesystem for the given file descriptor. +#[cfg_attr(target_os = "macos", allow(unused_variables))] pub fn syncfs(fd: impl AsRawFd) -> anyhow::Result<()> { // Linux guarantees durability for syncfs. // POSIX doesn't have syncfs, and further does not actually guarantee durability of sync(). #[cfg(target_os = "linux")] { + use anyhow::Context; nix::unistd::syncfs(fd.as_raw_fd()).context("syncfs")?; } #[cfg(target_os = "macos")]