mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-15 17:32:56 +00:00
a74b60066c
## Problem Storage controller was observed to have unexpectedly large memory consumption when loaded with many thousands of shards. This was recently fixed: - https://github.com/neondatabase/neon/pull/7493 ...but we need a general test that the controller is well behaved with thousands of shards. Closes: https://github.com/neondatabase/neon/issues/7460 Closes: https://github.com/neondatabase/neon/issues/7463 ## Summary of changes - Add test test_storage_controller_many_tenants to exercise the system's behaviour with a more substantial workload. This test measures memory consumption and reproduces #7460 before the other changes in this PR. - Tweak reconcile_all's return value to make it nonzero if it spawns no reconcilers, but _would_ have spawned some reconcilers if they weren't blocked by the reconcile concurrency limit. This makes the test's reconcile_until_idle behave as expected (i.e. not complete until the system is nice and calm). - Fix an issue where tenant migrations would leave a spurious secondary location when migrated to some location that was not already their secondary (this was an existing low-impact bug that tripped up the test's consistency checks). On the test with 8000 shards, the resident memory per shard is about 20KiB. This is not really per-shard memory: the primary source of memory growth is the number of concurrent network/db clients we create. With 8000 shards, the test takes 125s to run on my workstation.
46 lines
1.1 KiB
TOML
46 lines
1.1 KiB
TOML
[package]
|
|
name = "control_plane"
|
|
version = "0.1.0"
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
|
|
[dependencies]
|
|
anyhow.workspace = true
|
|
async-trait.workspace = true
|
|
camino.workspace = true
|
|
clap.workspace = true
|
|
comfy-table.workspace = true
|
|
futures.workspace = true
|
|
git-version.workspace = true
|
|
humantime.workspace = true
|
|
nix.workspace = true
|
|
once_cell.workspace = true
|
|
postgres.workspace = true
|
|
hex.workspace = true
|
|
humantime-serde.workspace = true
|
|
hyper.workspace = true
|
|
regex.workspace = true
|
|
reqwest = { workspace = true, features = ["blocking", "json"] }
|
|
scopeguard.workspace = true
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
serde_with.workspace = true
|
|
tar.workspace = true
|
|
thiserror.workspace = true
|
|
toml.workspace = true
|
|
tokio.workspace = true
|
|
tokio-postgres.workspace = true
|
|
tokio-util.workspace = true
|
|
url.workspace = true
|
|
pageserver_api.workspace = true
|
|
pageserver_client.workspace = true
|
|
postgres_backend.workspace = true
|
|
safekeeper_api.workspace = true
|
|
postgres_connection.workspace = true
|
|
storage_broker.workspace = true
|
|
utils.workspace = true
|
|
|
|
compute_api.workspace = true
|
|
workspace_hack.workspace = true
|
|
tracing.workspace = true
|