diff --git a/control_plane/src/pageserver.rs b/control_plane/src/pageserver.rs index 958f873b6a..dd6c8cf0a3 100644 --- a/control_plane/src/pageserver.rs +++ b/control_plane/src/pageserver.rs @@ -74,110 +74,6 @@ impl PageServerNode { } } - /// Merge overrides provided by the user on the command line with our default overides derived from neon_local configuration. - /// - /// These all end up on the command line of the `pageserver` binary. - fn neon_local_overrides(&self) -> Vec { - // FIXME: the paths should be shell-escaped to handle paths with spaces, quotas etc. - let pg_distrib_dir_param = format!( - "pg_distrib_dir='{}'", - self.env.pg_distrib_dir_raw().display() - ); - - let PageServerConf { - id, - listen_pg_addr, - listen_http_addr, - pg_auth_type, - http_auth_type, - virtual_file_io_engine, - get_vectored_impl, - get_impl, - validate_vectored_get, - } = &self.conf; - - let id = format!("id={}", id); - - let http_auth_type_param = format!("http_auth_type='{}'", http_auth_type); - let listen_http_addr_param = format!("listen_http_addr='{}'", listen_http_addr); - - let pg_auth_type_param = format!("pg_auth_type='{}'", pg_auth_type); - let listen_pg_addr_param = format!("listen_pg_addr='{}'", listen_pg_addr); - let virtual_file_io_engine = if let Some(virtual_file_io_engine) = virtual_file_io_engine { - format!("virtual_file_io_engine='{virtual_file_io_engine}'") - } else { - String::new() - }; - let get_vectored_impl = if let Some(get_vectored_impl) = get_vectored_impl { - format!("get_vectored_impl='{get_vectored_impl}'") - } else { - String::new() - }; - let get_impl = if let Some(get_impl) = get_impl { - format!("get_impl='{get_impl}'") - } else { - String::new() - }; - let validate_vectored_get = if let Some(validate_vectored_get) = validate_vectored_get { - format!("validate_vectored_get={validate_vectored_get}") - } else { - String::new() - }; - - let broker_endpoint_param = format!("broker_endpoint='{}'", self.env.broker.client_url()); - - let mut overrides = vec![ - id, - pg_distrib_dir_param, - http_auth_type_param, - pg_auth_type_param, - listen_http_addr_param, - listen_pg_addr_param, - broker_endpoint_param, - virtual_file_io_engine, - get_vectored_impl, - get_impl, - validate_vectored_get, - ]; - - if let Some(control_plane_api) = &self.env.control_plane_api { - overrides.push(format!( - "control_plane_api='{}'", - control_plane_api.as_str() - )); - - // Storage controller uses the same auth as pageserver: if JWT is enabled - // for us, we will also need it to talk to them. - if matches!(http_auth_type, AuthType::NeonJWT) { - let jwt_token = self - .env - .generate_auth_token(&Claims::new(None, Scope::GenerationsApi)) - .unwrap(); - overrides.push(format!("control_plane_api_token='{}'", jwt_token)); - } - } - - if !cli_overrides - .iter() - .any(|c| c.starts_with("remote_storage")) - { - overrides.push(format!( - "remote_storage={{local_path='../{PAGESERVER_REMOTE_STORAGE_DIR}'}}" - )); - } - - if *http_auth_type != AuthType::Trust || *pg_auth_type != AuthType::Trust { - // Keys are generated in the toplevel repo dir, pageservers' workdirs - // are one level below that, so refer to keys with ../ - overrides.push("auth_validation_public_key_path='../auth_public_key.pem'".to_owned()); - } - - // Apply the user-provided overrides - overrides.extend(cli_overrides.iter().map(|&c| c.to_owned())); - - overrides - } - /// Initializes a pageserver node by creating its config with the overrides provided. pub fn initialize(&self) -> anyhow::Result<()> { // First, run `pageserver --init` and wait for it to write a config into FS and exit. @@ -219,7 +115,124 @@ impl PageServerNode { let datadir_path_str = datadir.to_str().with_context(|| { format!("Cannot start pageserver node {node_id} in path that has no string representation: {datadir:?}") })?; - let mut args = self.pageserver_basic_args(datadir_path_str); + let mut args = { + let this = &self; + let mut args = vec![Cow::Borrowed("-D"), Cow::Borrowed(datadir_path_str)]; + + let overrides = { + let this = &this; + // FIXME: the paths should be shell-escaped to handle paths with spaces, quotas etc. + let pg_distrib_dir_param = format!( + "pg_distrib_dir='{}'", + this.env.pg_distrib_dir_raw().display() + ); + + let PageServerConf { + id, + listen_pg_addr, + listen_http_addr, + pg_auth_type, + http_auth_type, + virtual_file_io_engine, + get_vectored_impl, + get_impl, + validate_vectored_get, + } = &this.conf; + + let id = format!("id={}", id); + + let http_auth_type_param = format!("http_auth_type='{}'", http_auth_type); + let listen_http_addr_param = format!("listen_http_addr='{}'", listen_http_addr); + + let pg_auth_type_param = format!("pg_auth_type='{}'", pg_auth_type); + let listen_pg_addr_param = format!("listen_pg_addr='{}'", listen_pg_addr); + let virtual_file_io_engine = + if let Some(virtual_file_io_engine) = virtual_file_io_engine { + format!("virtual_file_io_engine='{virtual_file_io_engine}'") + } else { + String::new() + }; + let get_vectored_impl = if let Some(get_vectored_impl) = get_vectored_impl { + format!("get_vectored_impl='{get_vectored_impl}'") + } else { + String::new() + }; + let get_impl = if let Some(get_impl) = get_impl { + format!("get_impl='{get_impl}'") + } else { + String::new() + }; + let validate_vectored_get = + if let Some(validate_vectored_get) = validate_vectored_get { + format!("validate_vectored_get={validate_vectored_get}") + } else { + String::new() + }; + + let broker_endpoint_param = + format!("broker_endpoint='{}'", this.env.broker.client_url()); + + let mut overrides = vec![ + id, + pg_distrib_dir_param, + http_auth_type_param, + pg_auth_type_param, + listen_http_addr_param, + listen_pg_addr_param, + broker_endpoint_param, + virtual_file_io_engine, + get_vectored_impl, + get_impl, + validate_vectored_get, + ]; + + if let Some(control_plane_api) = &this.env.control_plane_api { + overrides.push(format!( + "control_plane_api='{}'", + control_plane_api.as_str() + )); + + // Storage controller uses the same auth as pageserver: if JWT is enabled + // for us, we will also need it to talk to them. + if matches!(http_auth_type, AuthType::NeonJWT) { + let jwt_token = this + .env + .generate_auth_token(&Claims::new(None, Scope::GenerationsApi)) + .unwrap(); + overrides.push(format!("control_plane_api_token='{}'", jwt_token)); + } + } + + if !config_overrides + .iter() + .any(|c| c.starts_with("remote_storage")) + { + overrides.push(format!( + "remote_storage={{local_path='../{PAGESERVER_REMOTE_STORAGE_DIR}'}}" + )); + } + + if *http_auth_type != AuthType::Trust || *pg_auth_type != AuthType::Trust { + // Keys are generated in the toplevel repo dir, pageservers' workdirs + // are one level below that, so refer to keys with ../ + overrides.push( + "auth_validation_public_key_path='../auth_public_key.pem'".to_owned(), + ); + } + + // Apply the user-provided overrides + overrides.extend(config_overrides.iter().map(|&c| c.to_owned())); + + overrides + }; + + for config_override in overrides { + args.push(Cow::Borrowed("--config-override")); + args.push(Cow::Owned(config_override)); + } + + args + }; args.push(Cow::Borrowed("--init")); let init_output = Command::new(self.env.pageserver_bin()) @@ -279,12 +292,16 @@ impl PageServerNode { self.conf.id, datadir, ) })?; - let args = self.pageserver_basic_args(datadir_path_str); + let mut args = vec!["-D", datadir_path_str]; + for config_override in config_overrides { + args.push("--config-override"); + args.push(*config_override); + } background_process::start_process( "pageserver", &datadir, &self.env.pageserver_bin(), - args.iter().map(Cow::as_ref), + args, self.pageserver_env_variables()?, background_process::InitialPidFile::Expect(self.pid_file()), || async { @@ -301,18 +318,6 @@ impl PageServerNode { Ok(()) } - fn pageserver_basic_args<'a>(&self, datadir_path_str: &'a str) -> Vec> { - let mut args = vec![Cow::Borrowed("-D"), Cow::Borrowed(datadir_path_str)]; - - let overrides = self.neon_local_overrides(); - for config_override in overrides { - args.push(Cow::Borrowed("-c")); - args.push(Cow::Owned(config_override)); - } - - args - } - fn pageserver_env_variables(&self) -> anyhow::Result> { // FIXME: why is this tied to pageserver's auth type? Whether or not the safekeeper // needs a token, and how to generate that token, seems independent to whether diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 7416eac76d..c3e16385e7 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -451,7 +451,7 @@ class NeonEnvBuilder: test_overlay_dir: Optional[Path] = None, pageserver_remote_storage: Optional[RemoteStorage] = None, # toml that will be decomposed into `--config-override` flags during `pageserver --init` - pageserver_init_overrides: Optional[str] = None, + pageserver_config_override: Optional[str] = None, num_safekeepers: int = 1, num_pageservers: int = 1, # Use non-standard SK ids to check for various parsing bugs @@ -478,7 +478,7 @@ class NeonEnvBuilder: self.broker = broker self.run_id = run_id self.mock_s3_server: MockS3Server = mock_s3_server - self.pageserver_init_overrides = pageserver_init_overrides + self.pageserver_config_override = pageserver_config_override self.num_safekeepers = num_safekeepers self.num_pageservers = num_pageservers self.safekeepers_id_start = safekeepers_id_start @@ -1134,7 +1134,7 @@ class NeonEnv: self.neon_cli.init( cfg, force=config.config_init_force, - pageserver_init_overrides=config.pageserver_init_overrides, + pageserver_config_override=config.pageserver_config_override, ) def start(self): @@ -1707,7 +1707,7 @@ class NeonCli(AbstractNeonCli): self, config: Dict[str, Any], force: Optional[str] = None, - pageserver_init_overrides: Optional[str] = None, + pageserver_config_override: Optional[str] = None, ) -> "subprocess.CompletedProcess[str]": with tempfile.NamedTemporaryFile(mode="w+") as tmp: tmp.write(toml.dumps(config)) @@ -1718,17 +1718,30 @@ class NeonCli(AbstractNeonCli): if force is not None: cmd.extend(["--force", force]) - storage = self.env.pageserver_remote_storage + remote_storage = self.env.pageserver_remote_storage - append_pageserver_param_overrides( - params_to_update=cmd, - remote_storage=storage, - pageserver_config_override=pageserver_init_overrides, - ) + if remote_storage is not None: + remote_storage_toml_table = remote_storage_to_toml_inline_table(remote_storage) + + cmd.append( + f"--pageserver-config-override=remote_storage={remote_storage_toml_table}" + ) + + env_overrides = os.getenv("NEON_PAGESERVER_OVERRIDES") + if env_overrides is not None: + cmd += [ + f"--pageserver-config-override={o.strip()}" for o in env_overrides.split(";") + ] + + if pageserver_config_override is not None: + cmd += [ + f"--pageserver-config-override={o.strip()}" + for o in pageserver_config_override.split(";") + ] s3_env_vars = None - if isinstance(storage, S3Storage): - s3_env_vars = storage.access_env_vars() + if isinstance(remote_storage, S3Storage): + s3_env_vars = remote_storage.access_env_vars() res = self.raw_cli(cmd, extra_env_vars=s3_env_vars) res.check_returncode() return res @@ -1750,10 +1763,6 @@ class NeonCli(AbstractNeonCli): ) -> "subprocess.CompletedProcess[str]": start_args = ["pageserver", "start", f"--id={id}"] storage = self.env.pageserver_remote_storage - append_pageserver_param_overrides( - params_to_update=start_args, - remote_storage=storage, - ) if isinstance(storage, S3Storage): s3_env_vars = storage.access_env_vars() @@ -2624,27 +2633,6 @@ class NeonPageserver(PgProtocol, LogUtils): ) -def append_pageserver_param_overrides( - params_to_update: List[str], - remote_storage: Optional[RemoteStorage], - pageserver_config_override: Optional[str] = None, -): - if remote_storage is not None: - remote_storage_toml_table = remote_storage_to_toml_inline_table(remote_storage) - - params_to_update.append( - f"--pageserver-config-override=remote_storage={remote_storage_toml_table}" - ) - else: - params_to_update.append('--pageserver-config-override=remote_storage=""') - - if pageserver_config_override is not None: - params_to_update += [ - f"--pageserver-config-override={o.strip()}" - for o in pageserver_config_override.split(";") - ] - - class PgBin: """A helper class for executing postgres binaries""" diff --git a/test_runner/performance/pageserver/pagebench/test_large_slru_basebackup.py b/test_runner/performance/pageserver/pagebench/test_large_slru_basebackup.py index 2a5d8b719e..b66db4d0ab 100644 --- a/test_runner/performance/pageserver/pagebench/test_large_slru_basebackup.py +++ b/test_runner/performance/pageserver/pagebench/test_large_slru_basebackup.py @@ -45,7 +45,7 @@ def test_basebackup_with_high_slru_count( # configure cache sizes like in prod page_cache_size = 16384 max_file_descriptors = 500000 - neon_env_builder.pageserver_init_overrides = ( + neon_env_builder.pageserver_config_override = ( f"page_cache_size={page_cache_size}; max_file_descriptors={max_file_descriptors}; " f"get_vectored_impl='{get_vectored_impl}'; validate_vectored_get=false" ) diff --git a/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py b/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py index 24394e1c16..1a0012397c 100644 --- a/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py +++ b/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py @@ -66,7 +66,7 @@ def test_pageserver_max_throughput_getpage_at_latest_lsn( # configure cache sizes like in prod page_cache_size = 16384 max_file_descriptors = 500000 - neon_env_builder.pageserver_init_overrides = ( + neon_env_builder.pageserver_config_override = ( f"page_cache_size={page_cache_size}; max_file_descriptors={max_file_descriptors}" ) params.update( diff --git a/test_runner/regress/test_branch_behind.py b/test_runner/regress/test_branch_behind.py index 07d8bc277b..b79cad979f 100644 --- a/test_runner/regress/test_branch_behind.py +++ b/test_runner/regress/test_branch_behind.py @@ -11,7 +11,7 @@ from fixtures.utils import print_gc_result, query_scalar # def test_branch_behind(neon_env_builder: NeonEnvBuilder): # Disable pitr, because here we want to test branch creation after GC - neon_env_builder.pageserver_init_overrides = "tenant_config={pitr_interval = '0 sec'}" + neon_env_builder.pageserver_config_override = "tenant_config={pitr_interval = '0 sec'}" env = neon_env_builder.init_start() error_regexes = [ diff --git a/test_runner/regress/test_compaction.py b/test_runner/regress/test_compaction.py index 1e832f90b4..43a3323462 100644 --- a/test_runner/regress/test_compaction.py +++ b/test_runner/regress/test_compaction.py @@ -31,7 +31,7 @@ def test_pageserver_compaction_smoke(neon_env_builder: NeonEnvBuilder): # Effectively disable the page cache to rely only on image layers # to shorten reads. - neon_env_builder.pageserver_init_overrides = """ + neon_env_builder.pageserver_config_override = """ page_cache_size=10 """ diff --git a/test_runner/regress/test_gc_aggressive.py b/test_runner/regress/test_gc_aggressive.py index 3faa8cc53c..c5070ee815 100644 --- a/test_runner/regress/test_gc_aggressive.py +++ b/test_runner/regress/test_gc_aggressive.py @@ -67,7 +67,7 @@ async def update_and_gc(env: NeonEnv, endpoint: Endpoint, timeline: TimelineId): # def test_gc_aggressive(neon_env_builder: NeonEnvBuilder): # Disable pitr, because here we want to test branch creation after GC - neon_env_builder.pageserver_init_overrides = "tenant_config={pitr_interval = '0 sec'}" + neon_env_builder.pageserver_config_override = "tenant_config={pitr_interval = '0 sec'}" env = neon_env_builder.init_start() timeline = env.neon_cli.create_branch("test_gc_aggressive", "main") endpoint = env.endpoints.create_start("test_gc_aggressive") @@ -95,7 +95,7 @@ def test_gc_aggressive(neon_env_builder: NeonEnvBuilder): # def test_gc_index_upload(neon_env_builder: NeonEnvBuilder): # Disable time-based pitr, we will use LSN-based thresholds in the manual GC calls - neon_env_builder.pageserver_init_overrides = "tenant_config={pitr_interval = '0 sec'}" + neon_env_builder.pageserver_config_override = "tenant_config={pitr_interval = '0 sec'}" num_index_uploads = 0 neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) diff --git a/test_runner/regress/test_local_file_cache.py b/test_runner/regress/test_local_file_cache.py index 710bd4c847..76c6581448 100644 --- a/test_runner/regress/test_local_file_cache.py +++ b/test_runner/regress/test_local_file_cache.py @@ -11,7 +11,7 @@ from fixtures.utils import query_scalar def test_local_file_cache_unlink(neon_env_builder: NeonEnvBuilder, build_type: str): if build_type == "debug": # Disable vectored read path cross validation since it makes the test time out. - neon_env_builder.pageserver_init_overrides = "validate_vectored_get=false" + neon_env_builder.pageserver_config_override = "validate_vectored_get=false" env = neon_env_builder.init_start() diff --git a/test_runner/regress/test_old_request_lsn.py b/test_runner/regress/test_old_request_lsn.py index 6e8c302186..391305c58a 100644 --- a/test_runner/regress/test_old_request_lsn.py +++ b/test_runner/regress/test_old_request_lsn.py @@ -16,7 +16,7 @@ from fixtures.utils import print_gc_result, query_scalar # def test_old_request_lsn(neon_env_builder: NeonEnvBuilder): # Disable pitr, because here we want to test branch creation after GC - neon_env_builder.pageserver_init_overrides = "tenant_config={pitr_interval = '0 sec'}" + neon_env_builder.pageserver_config_override = "tenant_config={pitr_interval = '0 sec'}" env = neon_env_builder.init_start() env.neon_cli.create_branch("test_old_request_lsn", "main") endpoint = env.endpoints.create_start("test_old_request_lsn") diff --git a/test_runner/regress/test_pageserver_config.py b/test_runner/regress/test_pageserver_config.py index fab62d226f..c04348b488 100644 --- a/test_runner/regress/test_pageserver_config.py +++ b/test_runner/regress/test_pageserver_config.py @@ -7,7 +7,7 @@ from fixtures.neon_fixtures import ( @pytest.mark.parametrize("kind", ["sync", "async"]) def test_walredo_process_kind_config(neon_env_builder: NeonEnvBuilder, kind: str): - neon_env_builder.pageserver_init_overrides = f"walredo_process_kind = '{kind}'" + neon_env_builder.pageserver_config_override = f"walredo_process_kind = '{kind}'" # ensure it starts env = neon_env_builder.init_start() # ensure the metric is set diff --git a/test_runner/regress/test_pageserver_layer_rolling.py b/test_runner/regress/test_pageserver_layer_rolling.py index 16f69d8259..c5dc0f2919 100644 --- a/test_runner/regress/test_pageserver_layer_rolling.py +++ b/test_runner/regress/test_pageserver_layer_rolling.py @@ -241,7 +241,7 @@ def test_total_size_limit(neon_env_builder: NeonEnvBuilder): ephemeral_bytes_per_memory_kb = (max_dirty_data * 1024) // system_memory assert ephemeral_bytes_per_memory_kb > 0 - neon_env_builder.pageserver_init_overrides = f""" + neon_env_builder.pageserver_config_override = f""" ephemeral_bytes_per_memory_kb={ephemeral_bytes_per_memory_kb} """ diff --git a/test_runner/regress/test_pageserver_metric_collection.py b/test_runner/regress/test_pageserver_metric_collection.py index b5973e5241..c34ef46d07 100644 --- a/test_runner/regress/test_pageserver_metric_collection.py +++ b/test_runner/regress/test_pageserver_metric_collection.py @@ -54,7 +54,7 @@ def test_metric_collection( # # Disable time-based pitr, we will use the manual GC calls # to trigger remote storage operations in a controlled way - neon_env_builder.pageserver_init_overrides = f""" + neon_env_builder.pageserver_config_override = f""" metric_collection_interval="1s" metric_collection_endpoint="{metric_collection_endpoint}" metric_collection_bucket={remote_storage_to_toml_inline_table(neon_env_builder.pageserver_remote_storage)} @@ -216,7 +216,7 @@ def test_metric_collection_cleans_up_tempfile( # # Disable time-based pitr, we will use the manual GC calls # to trigger remote storage operations in a controlled way - neon_env_builder.pageserver_init_overrides = f""" + neon_env_builder.pageserver_config_override = f""" metric_collection_interval="1s" metric_collection_endpoint="{metric_collection_endpoint}" cached_metric_collection_interval="0s" diff --git a/test_runner/regress/test_pg_regress.py b/test_runner/regress/test_pg_regress.py index f0debba00f..2b1b7fff34 100644 --- a/test_runner/regress/test_pg_regress.py +++ b/test_runner/regress/test_pg_regress.py @@ -34,7 +34,7 @@ def test_pg_regress( if build_type == "debug": # Disable vectored read path cross validation since it makes the test time out. - neon_env_builder.pageserver_init_overrides = "validate_vectored_get=false" + neon_env_builder.pageserver_config_override = "validate_vectored_get=false" neon_env_builder.enable_pageserver_remote_storage(s3_storage()) neon_env_builder.enable_scrub_on_exit() diff --git a/test_runner/regress/test_pitr_gc.py b/test_runner/regress/test_pitr_gc.py index a2c37cc0f5..539ef3eda7 100644 --- a/test_runner/regress/test_pitr_gc.py +++ b/test_runner/regress/test_pitr_gc.py @@ -10,7 +10,7 @@ from fixtures.utils import print_gc_result, query_scalar # def test_pitr_gc(neon_env_builder: NeonEnvBuilder): # Set pitr interval such that we need to keep the data - neon_env_builder.pageserver_init_overrides = ( + neon_env_builder.pageserver_config_override = ( "tenant_config={pitr_interval = '1 day', gc_horizon = 0}" ) diff --git a/test_runner/regress/test_recovery.py b/test_runner/regress/test_recovery.py index bdcf03f27a..ab5c8be256 100644 --- a/test_runner/regress/test_recovery.py +++ b/test_runner/regress/test_recovery.py @@ -10,7 +10,7 @@ from fixtures.neon_fixtures import NeonEnvBuilder # def test_pageserver_recovery(neon_env_builder: NeonEnvBuilder): # Override default checkpointer settings to run it more often - neon_env_builder.pageserver_init_overrides = "tenant_config={checkpoint_distance = 1048576}" + neon_env_builder.pageserver_config_override = "tenant_config={checkpoint_distance = 1048576}" env = neon_env_builder.init_start() env.pageserver.is_testing_enabled_or_skip() diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py index 07a0a4fd98..ad4b4a42f1 100644 --- a/test_runner/regress/test_remote_storage.py +++ b/test_runner/regress/test_remote_storage.py @@ -67,7 +67,7 @@ def test_remote_storage_backup_and_restore( # Exercise retry code path by making all uploads and downloads fail for the # first time. The retries print INFO-messages to the log; we will check # that they are present after the test. - neon_env_builder.pageserver_init_overrides = "test_remote_failures=1" + neon_env_builder.pageserver_config_override = "test_remote_failures=1" data_id = 1 data = "just some data" diff --git a/test_runner/regress/test_tenant_conf.py b/test_runner/regress/test_tenant_conf.py index 9c4385f999..fc099297e1 100644 --- a/test_runner/regress/test_tenant_conf.py +++ b/test_runner/regress/test_tenant_conf.py @@ -15,7 +15,7 @@ from fixtures.utils import wait_until def test_tenant_config(neon_env_builder: NeonEnvBuilder): """Test per tenant configuration""" # set some non-default global config - neon_env_builder.pageserver_init_overrides = """ + neon_env_builder.pageserver_config_override = """ page_cache_size=444; wait_lsn_timeout='111 s'; [tenant_config] diff --git a/test_runner/regress/test_tenant_delete.py b/test_runner/regress/test_tenant_delete.py index 22df441362..c115c0375b 100644 --- a/test_runner/regress/test_tenant_delete.py +++ b/test_runner/regress/test_tenant_delete.py @@ -35,7 +35,7 @@ def test_tenant_delete_smoke( neon_env_builder: NeonEnvBuilder, pg_bin: PgBin, ): - neon_env_builder.pageserver_init_overrides = "test_remote_failures=1" + neon_env_builder.pageserver_config_override = "test_remote_failures=1" remote_storage_kind = s3_storage() neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) @@ -169,7 +169,7 @@ def test_delete_tenant_exercise_crash_safety_failpoints( pg_bin: PgBin, ): if simulate_failures: - neon_env_builder.pageserver_init_overrides = "test_remote_failures=1" + neon_env_builder.pageserver_config_override = "test_remote_failures=1" neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py index 30083d931c..0ba0108651 100644 --- a/test_runner/regress/test_tenant_detach.py +++ b/test_runner/regress/test_tenant_detach.py @@ -66,7 +66,7 @@ def test_tenant_reattach(neon_env_builder: NeonEnvBuilder, mode: str): # Exercise retry code path by making all uploads and downloads fail for the # first time. The retries print INFO-messages to the log; we will check # that they are present after the test. - neon_env_builder.pageserver_init_overrides = "test_remote_failures=1" + neon_env_builder.pageserver_config_override = "test_remote_failures=1" env = neon_env_builder.init_start() pageserver_http = env.pageserver.http_client() diff --git a/test_runner/regress/test_tenant_size.py b/test_runner/regress/test_tenant_size.py index f7c020dd81..53da548524 100644 --- a/test_runner/regress/test_tenant_size.py +++ b/test_runner/regress/test_tenant_size.py @@ -502,7 +502,7 @@ def test_get_tenant_size_with_multiple_branches( gc_horizon = 128 * 1024 - neon_env_builder.pageserver_init_overrides = f"tenant_config={{compaction_period='0s', gc_period='0s', pitr_interval='0sec', gc_horizon={gc_horizon}}}" + neon_env_builder.pageserver_config_override = f"tenant_config={{compaction_period='0s', gc_period='0s', pitr_interval='0sec', gc_horizon={gc_horizon}}}" env = neon_env_builder.init_start() diff --git a/test_runner/regress/test_tenants.py b/test_runner/regress/test_tenants.py index 92c6553530..2832304dcc 100644 --- a/test_runner/regress/test_tenants.py +++ b/test_runner/regress/test_tenants.py @@ -89,7 +89,7 @@ def test_tenants_normal_work(neon_env_builder: NeonEnvBuilder): def test_metrics_normal_work(neon_env_builder: NeonEnvBuilder): neon_env_builder.num_safekeepers = 3 - neon_env_builder.pageserver_init_overrides = "availability_zone='test_ps_az'" + neon_env_builder.pageserver_config_override = "availability_zone='test_ps_az'" env = neon_env_builder.init_start() tenant_1, _ = env.neon_cli.create_tenant() diff --git a/test_runner/regress/test_tenants_with_remote_storage.py b/test_runner/regress/test_tenants_with_remote_storage.py index e8a0d2e25b..d16978d02a 100644 --- a/test_runner/regress/test_tenants_with_remote_storage.py +++ b/test_runner/regress/test_tenants_with_remote_storage.py @@ -107,7 +107,7 @@ def test_tenants_attached_after_download(neon_env_builder: NeonEnvBuilder): # Exercise retry code path by making all uploads and downloads fail for the # first time. The retries print INFO-messages to the log; we will check # that they are present after the test. - neon_env_builder.pageserver_init_overrides = "test_remote_failures=1" + neon_env_builder.pageserver_config_override = "test_remote_failures=1" ##### First start, insert secret data and upload it to the remote storage env = neon_env_builder.init_start() diff --git a/test_runner/regress/test_threshold_based_eviction.py b/test_runner/regress/test_threshold_based_eviction.py index 8063b09ce9..7bf49a0874 100644 --- a/test_runner/regress/test_threshold_based_eviction.py +++ b/test_runner/regress/test_threshold_based_eviction.py @@ -27,7 +27,7 @@ def test_threshold_based_eviction( # imitates its accesses. We'll use a non-existent endpoint to make it fail. # The synthetic size calculation will run regardless. host, port = httpserver_listen_address - neon_env_builder.pageserver_init_overrides = f""" + neon_env_builder.pageserver_config_override = f""" metric_collection_interval="1s" synthetic_size_calculation_interval="2s" metric_collection_endpoint="http://{host}:{port}/nonexistent" diff --git a/test_runner/regress/test_timeline_size.py b/test_runner/regress/test_timeline_size.py index 4a6500414d..628c484fbd 100644 --- a/test_runner/regress/test_timeline_size.py +++ b/test_runner/regress/test_timeline_size.py @@ -415,7 +415,7 @@ def test_timeline_physical_size_post_compaction(neon_env_builder: NeonEnvBuilder # Disable background compaction as we don't want it to happen after `get_physical_size` request # and before checking the expected size on disk, which makes the assertion failed - neon_env_builder.pageserver_init_overrides = ( + neon_env_builder.pageserver_config_override = ( "tenant_config={checkpoint_distance=100000, compaction_period='10m'}" ) @@ -462,7 +462,7 @@ def test_timeline_physical_size_post_gc(neon_env_builder: NeonEnvBuilder): # Disable background compaction and GC as we don't want it to happen after `get_physical_size` request # and before checking the expected size on disk, which makes the assertion failed - neon_env_builder.pageserver_init_overrides = "tenant_config={checkpoint_distance=100000, compaction_period='0s', gc_period='0s', pitr_interval='1s'}" + neon_env_builder.pageserver_config_override = "tenant_config={checkpoint_distance=100000, compaction_period='0s', gc_period='0s', pitr_interval='1s'}" env = neon_env_builder.init_start() pageserver_http = env.pageserver.http_client() @@ -704,7 +704,7 @@ def test_ondemand_activation(neon_env_builder: NeonEnvBuilder): # We will run with the limit set to 1, so that once we have one tenant stuck # in a pausable failpoint, the rest are prevented from proceeding through warmup. - neon_env_builder.pageserver_init_overrides = "concurrent_tenant_warmup = '1'" + neon_env_builder.pageserver_config_override = "concurrent_tenant_warmup = '1'" env = neon_env_builder.init_start() pageserver_http = env.pageserver.http_client() @@ -958,7 +958,7 @@ def test_timeline_logical_size_task_priority(neon_env_builder: NeonEnvBuilder): def test_eager_attach_does_not_queue_up(neon_env_builder: NeonEnvBuilder): - neon_env_builder.pageserver_init_overrides = "concurrent_tenant_warmup = '1'" + neon_env_builder.pageserver_config_override = "concurrent_tenant_warmup = '1'" env = neon_env_builder.init_start() @@ -1036,7 +1036,7 @@ def test_eager_attach_does_not_queue_up(neon_env_builder: NeonEnvBuilder): @pytest.mark.parametrize("activation_method", ["endpoint", "branch", "delete"]) def test_lazy_attach_activation(neon_env_builder: NeonEnvBuilder, activation_method: str): # env.initial_tenant will take up this permit when attaching with lazy because of a failpoint activated after restart - neon_env_builder.pageserver_init_overrides = "concurrent_tenant_warmup = '1'" + neon_env_builder.pageserver_config_override = "concurrent_tenant_warmup = '1'" env = neon_env_builder.init_start() diff --git a/test_runner/regress/test_wal_receiver.py b/test_runner/regress/test_wal_receiver.py index 894c9c03a8..7ac6e6332c 100644 --- a/test_runner/regress/test_wal_receiver.py +++ b/test_runner/regress/test_wal_receiver.py @@ -9,7 +9,7 @@ from fixtures.types import Lsn, TenantId # Ensures that walreceiver does not run without any data inserted and only starts after the insertion. def test_pageserver_lsn_wait_error_start(neon_env_builder: NeonEnvBuilder): # Trigger WAL wait timeout faster - neon_env_builder.pageserver_init_overrides = "wait_lsn_timeout = '1s'" + neon_env_builder.pageserver_config_override = "wait_lsn_timeout = '1s'" env = neon_env_builder.init_start() env.pageserver.http_client() @@ -42,7 +42,7 @@ def test_pageserver_lsn_wait_error_start(neon_env_builder: NeonEnvBuilder): # Kills one of the safekeepers and ensures that only the active ones are printed in the state. def test_pageserver_lsn_wait_error_safekeeper_stop(neon_env_builder: NeonEnvBuilder): # Trigger WAL wait timeout faster - neon_env_builder.pageserver_init_overrides = """ + neon_env_builder.pageserver_config_override = """ wait_lsn_timeout = "1s" tenant_config={walreceiver_connect_timeout = "2s", lagging_wal_timeout = "2s"} """