From 9df230c837691d526c3d37d8e167fc492a936baf Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Wed, 2 Apr 2025 17:11:52 +0200 Subject: [PATCH] storcon: improve autosplit defaults (#11332) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem In #11122, we changed the autosplit behavior to allow repeated and initial splits. The defaults were set such that they retain the current production settings (8 shards at 64 GB). However, these defaults don't really make sense by themselves. Once we deploy new settings to production, we should change the defaults to something more reasonable. ## Summary of changes Changes the following default settings: * `max_split_shards`: 8 → 16 * `initial_split_threshold`: 64 GB → disabled * `initial_split_shards`: 8 → 2 --- storage_controller/src/main.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/storage_controller/src/main.rs b/storage_controller/src/main.rs index 1a7f9a2366..8c834f9acb 100644 --- a/storage_controller/src/main.rs +++ b/storage_controller/src/main.rs @@ -115,19 +115,17 @@ struct Cli { #[arg(long)] split_threshold: Option, - /// Maximum number of shards during autosplits. 0 disables autosplits. - // TODO: defaults to 8 for backwards compatibility, should default to 255. - #[arg(long, default_value = "8")] + /// Maximum number of shards during autosplits. 0 disables autosplits. Defaults + /// to 16 as a safety to avoid too many shards by accident. + #[arg(long, default_value = "16")] max_split_shards: u8, /// Size threshold for initial shard splits of unsharded tenants. 0 disables initial splits. - // TODO: defaults to 64 GB for backwards compatibility. Should default to None. - #[arg(long, default_value = "68719476736")] - initial_split_threshold: u64, + #[arg(long)] + initial_split_threshold: Option, - /// Number of target shards for initial splits. 0 or 1 disables initial splits. - // TODO: defaults to 8 for backwards compatibility. Should default to 2. - #[arg(long, default_value = "8")] + /// Number of target shards for initial splits. 0 or 1 disables initial splits. Defaults to 2. + #[arg(long, default_value = "2")] initial_split_shards: u8, /// Maximum number of normal-priority reconcilers that may run in parallel @@ -417,7 +415,7 @@ async fn async_main() -> anyhow::Result<()> { tenant_rate_limit: args.tenant_rate_limit, split_threshold: args.split_threshold, max_split_shards: args.max_split_shards, - initial_split_threshold: Some(args.initial_split_threshold), + initial_split_threshold: args.initial_split_threshold, initial_split_shards: args.initial_split_shards, neon_local_repo_dir: args.neon_local_repo_dir, max_secondary_lag_bytes: args.max_secondary_lag_bytes,