mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-17 02:12:56 +00:00
850421ec06
This PR simplifies the pageserver configuration parsing as follows:
* introduce the `pageserver_api::config::ConfigToml` type
* implement `Default` for `ConfigToml`
* use serde derive to do the brain-dead leg-work of processing the toml
document
* use `serde(default)` to fill in default values
* in `pageserver` crate:
* use `toml_edit` to deserialize the pageserver.toml string into a
`ConfigToml`
* `PageServerConfig::parse_and_validate` then
* consumes the `ConfigToml`
* destructures it exhaustively into its constituent fields
* constructs the `PageServerConfig`
The rules are:
* in `ConfigToml`, use `deny_unknown_fields` everywhere
* static default values go in `pageserver_api`
* if there cannot be a static default value (e.g. which default IO
engine to use, because it depends on the runtime), make the field in
`ConfigToml` an `Option`
* if runtime-augmentation of a value is needed, do that in
`parse_and_validate`
* a good example is `virtual_file_io_engine` or `l0_flush`, both of
which need to execute code to determine the effective value in
`PageServerConf`
The benefits:
* massive amount of brain-dead repetitive code can be deleted
* "unused variable" compile-time errors when removing a config value,
due to the exhaustive destructuring in `parse_and_validate`
* compile-time errors guide you when adding a new config field
Drawbacks:
* serde derive is sometimes a bit too magical
* `deny_unknown_fields` is easy to miss
Future Work / Benefits:
* make `neon_local` use `pageserver_api` to construct `ConfigToml` and
write it to `pageserver.toml`
* This provides more type safety / coompile-time errors than the current
approach.
### Refs
Fixes #3682
### Future Work
* `remote_storage` deser doesn't reject unknown fields
https://github.com/neondatabase/neon/issues/8915
* clean up `libs/pageserver_api/src/config.rs` further
* break up into multiple files, at least for tenant config
* move `models` as appropriate / refine distinction between config and
API models / be explicit about when it's the same
* use `pub(crate)` visibility on `mod defaults` to detect stale values
48 lines
1.3 KiB
Rust
48 lines
1.3 KiB
Rust
use std::{num::NonZeroUsize, sync::Arc};
|
|
|
|
#[derive(Debug, PartialEq, Eq, Clone)]
|
|
pub enum L0FlushConfig {
|
|
Direct { max_concurrency: NonZeroUsize },
|
|
}
|
|
|
|
impl Default for L0FlushConfig {
|
|
fn default() -> Self {
|
|
Self::Direct {
|
|
// TODO: using num_cpus results in different peak memory usage on different instance types.
|
|
max_concurrency: NonZeroUsize::new(usize::max(1, num_cpus::get())).unwrap(),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl From<pageserver_api::models::L0FlushConfig> for L0FlushConfig {
|
|
fn from(config: pageserver_api::models::L0FlushConfig) -> Self {
|
|
match config {
|
|
pageserver_api::models::L0FlushConfig::Direct { max_concurrency } => {
|
|
Self::Direct { max_concurrency }
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
pub struct L0FlushGlobalState(Arc<Inner>);
|
|
|
|
pub enum Inner {
|
|
Direct { semaphore: tokio::sync::Semaphore },
|
|
}
|
|
|
|
impl L0FlushGlobalState {
|
|
pub fn new(config: L0FlushConfig) -> Self {
|
|
match config {
|
|
L0FlushConfig::Direct { max_concurrency } => {
|
|
let semaphore = tokio::sync::Semaphore::new(max_concurrency.get());
|
|
Self(Arc::new(Inner::Direct { semaphore }))
|
|
}
|
|
}
|
|
}
|
|
|
|
pub fn inner(&self) -> &Arc<Inner> {
|
|
&self.0
|
|
}
|
|
}
|