mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-07 21:42:56 +00:00
4d3c477689
This PR extends the storcon with basic safekeeper management of timelines, mainly timeline creation and deletion. We want to make the storcon manage safekeepers in the future. Timeline creation is controlled by the `--timelines-onto-safekeepers` flag. 1. it adds the `timelines` and `safekeeper_timeline_pending_ops` tables to the storcon db 2. extend code for the timeline creation and deletion 4. it adds per-safekeeper reconciler tasks TODO: * maybe not immediately schedule reconciliations for deletions but have a prior manual step * tenant deletions * add exclude API definitions (probably separate PR) * how to choose safekeeper to do exclude on vs deletion? this can be a bit hairy because the safekeeper might go offline in the meantime. * error/failure case handling * tests (cc test_explicit_timeline_creation from #11002) * single safekeeper mode: we often only have one SK (in tests for example) * `notify-safekeepers` hook: https://github.com/neondatabase/neon/issues/11163 TODOs implemented: * cancellations of enqueued reconciliations on a per-timeline basis, helpful if there is an ongoing deletion * implement pending ops overwrite behavior * load pending operations from db RFC section for important reading: [link](https://github.com/neondatabase/neon/blob/main/docs/rfcs/035-safekeeper-dynamic-membership-change.md#storage_controller-implementation) Implements the bulk of #9011 Successor of #10440. --------- Co-authored-by: Arseny Sher <sher-ars@yandex.ru>
70 lines
1.8 KiB
TOML
70 lines
1.8 KiB
TOML
[package]
|
|
name = "storage_controller"
|
|
version = "0.1.0"
|
|
edition = "2024"
|
|
license.workspace = true
|
|
|
|
[[bin]]
|
|
name = "storage_controller"
|
|
path = "src/main.rs"
|
|
|
|
[features]
|
|
default = []
|
|
# Enables test-only APIs and behaviors
|
|
testing = []
|
|
|
|
[dependencies]
|
|
anyhow.workspace = true
|
|
bytes.workspace = true
|
|
chrono.workspace = true
|
|
clap.workspace = true
|
|
cron.workspace = true
|
|
clashmap.workspace = true
|
|
fail.workspace = true
|
|
futures.workspace = true
|
|
governor.workspace = true
|
|
hex.workspace = true
|
|
hyper0.workspace = true
|
|
humantime.workspace = true
|
|
itertools.workspace = true
|
|
json-structural-diff.workspace = true
|
|
lasso.workspace = true
|
|
once_cell.workspace = true
|
|
pageserver_api.workspace = true
|
|
pageserver_client.workspace = true
|
|
postgres_connection.workspace = true
|
|
rand.workspace = true
|
|
reqwest = { workspace = true, features = ["stream"] }
|
|
routerify.workspace = true
|
|
safekeeper_api.workspace = true
|
|
safekeeper_client.workspace = true
|
|
tikv-jemallocator.workspace = true
|
|
regex.workspace = true
|
|
rustls-native-certs.workspace = true
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
thiserror.workspace = true
|
|
tokio.workspace = true
|
|
tokio-util.workspace = true
|
|
tracing.workspace = true
|
|
measured.workspace = true
|
|
rustls.workspace = true
|
|
scopeguard.workspace = true
|
|
strum.workspace = true
|
|
strum_macros.workspace = true
|
|
tokio-postgres.workspace = true
|
|
tokio-postgres-rustls.workspace = true
|
|
|
|
diesel = { version = "2.2.6", features = [
|
|
"serde_json",
|
|
"chrono",
|
|
] }
|
|
diesel-async = { version = "0.5.2", features = ["postgres", "bb8", "async-connection-wrapper"] }
|
|
diesel_migrations = { version = "2.2.0" }
|
|
scoped-futures = "0.1.4"
|
|
|
|
http-utils = { path = "../libs/http-utils/" }
|
|
utils = { path = "../libs/utils/" }
|
|
metrics = { path = "../libs/metrics/" }
|
|
control_plane = { path = "../control_plane" }
|
|
workspace_hack = { version = "0.1", path = "../workspace_hack" } |