mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-07 21:42:56 +00:00
1678dea20f
## Problem We recently added a "visibility" state to layers, but nothing initializes it. Part of: - #8398 ## Summary of changes - Add a dependency on `range-set-blaze`, which is used as a fast incrementally updated alternative to KeySpace. We could also use this to replace the internals of KeySpaceRandomAccum if we wanted to. Writing a type that does this kind of "BtreeMap & merge overlapping entries" thing isn't super complicated, but no reason to write this ourselves when there's a third party impl available. - Add a function to layermap to calculate visibilities for each layer - Add a function to Timeline to call into layermap and then apply these visibilities to the Layer objects. - Invoke the calculation during startup, after image layer creations, and when removing branches. Branch removal and image layer creation are the two ways that a layer can go from Visible to Covered. - Add unit test & benchmark for the visibility calculation - Expose `pageserver_visible_physical_size` metric, which should always be <= `pageserver_remote_physical_size`. - This metric will feed into the /v1/utilization endpoint later: the visible size indicates how much space we would like to use on this pageserver for this tenant. - When `pageserver_visible_physical_size` is greater than `pageserver_resident_physical_size`, this is a sign that the tenant has long-idle branches, which result in layers that are visible in principle, but not used in practice. This does not keep visibility hints up to date in all cases: particularly, when creating a child timeline, any previously covered layers will not get marked Visible until they are accessed. Updates after image layer creation could be implemented as more of a special case, but this would require more new code: the existing depth calculation code doesn't maintain+yield the list of deltas that would be covered by an image layer. ## Performance This operation is done rarely (at startup and at timeline deletion), so needs to be efficient but not ultra-fast. There is a new `visibility` bench that measures runtime for a synthetic 100k layers case (`sequential`) and a real layer map (`real_map`) with ~26k layers. The benchmark shows runtimes of single digit milliseconds (on a ryzen 7950). This confirms that the runtime shouldn't be a problem at startup (as we already incur S3-level latencies there), but that it's slow enough that we definitely shouldn't call it more often than necessary, and it may be worthwhile to optimize further later (things like: when removing a branch, only bother scanning layers below the branchpoint) ``` visibility/sequential time: [4.5087 ms 4.5894 ms 4.6775 ms] change: [+2.0826% +3.9097% +5.8995%] (p = 0.00 < 0.05) Performance has regressed. Found 24 outliers among 100 measurements (24.00%) 2 (2.00%) high mild 22 (22.00%) high severe min: 0/1696070, max: 93/1C0887F0 visibility/real_map time: [7.0796 ms 7.0832 ms 7.0871 ms] change: [+0.3900% +0.4505% +0.5164%] (p = 0.00 < 0.05) Change within noise threshold. Found 4 outliers among 100 measurements (4.00%) 3 (3.00%) high mild 1 (1.00%) high severe min: 0/1696070, max: 93/1C0887F0 visibility/real_map_many_branches time: [4.5285 ms 4.5355 ms 4.5434 ms] change: [-1.0012% -0.8004% -0.5969%] (p = 0.00 < 0.05) Change within noise threshold. ```
111 lines
3.2 KiB
TOML
111 lines
3.2 KiB
TOML
[package]
|
|
name = "pageserver"
|
|
version = "0.1.0"
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
|
|
[features]
|
|
default = []
|
|
# Enables test-only APIs, incuding failpoints. In particular, enables the `fail_point!` macro,
|
|
# which adds some runtime cost to run tests on outage conditions
|
|
testing = ["fail/failpoints"]
|
|
|
|
[dependencies]
|
|
anyhow.workspace = true
|
|
arc-swap.workspace = true
|
|
async-compression.workspace = true
|
|
async-stream.workspace = true
|
|
async-trait.workspace = true
|
|
byteorder.workspace = true
|
|
bytes.workspace = true
|
|
camino.workspace = true
|
|
camino-tempfile.workspace = true
|
|
chrono = { workspace = true, features = ["serde"] }
|
|
clap = { workspace = true, features = ["string"] }
|
|
const_format.workspace = true
|
|
consumption_metrics.workspace = true
|
|
crc32c.workspace = true
|
|
crossbeam-utils.workspace = true
|
|
either.workspace = true
|
|
flate2.workspace = true
|
|
fail.workspace = true
|
|
futures.workspace = true
|
|
git-version.workspace = true
|
|
hex.workspace = true
|
|
humantime.workspace = true
|
|
humantime-serde.workspace = true
|
|
hyper.workspace = true
|
|
itertools.workspace = true
|
|
leaky-bucket.workspace = true
|
|
md5.workspace = true
|
|
nix.workspace = true
|
|
# hack to get the number of worker threads tokio uses
|
|
num_cpus = { version = "1.15" }
|
|
num-traits.workspace = true
|
|
once_cell.workspace = true
|
|
pin-project-lite.workspace = true
|
|
postgres.workspace = true
|
|
postgres_backend.workspace = true
|
|
postgres-protocol.workspace = true
|
|
postgres-types.workspace = true
|
|
rand.workspace = true
|
|
range-set-blaze = { version = "0.1.16", features = ["alloc"] }
|
|
regex.workspace = true
|
|
scopeguard.workspace = true
|
|
serde.workspace = true
|
|
serde_json = { workspace = true, features = ["raw_value"] }
|
|
serde_path_to_error.workspace = true
|
|
serde_with.workspace = true
|
|
signal-hook.workspace = true
|
|
smallvec = { workspace = true, features = ["write"] }
|
|
svg_fmt.workspace = true
|
|
sync_wrapper.workspace = true
|
|
sysinfo.workspace = true
|
|
tokio-tar.workspace = true
|
|
thiserror.workspace = true
|
|
tikv-jemallocator.workspace = true
|
|
tokio = { workspace = true, features = ["process", "sync", "fs", "rt", "io-util", "time"] }
|
|
tokio-epoll-uring.workspace = true
|
|
tokio-io-timeout.workspace = true
|
|
tokio-postgres.workspace = true
|
|
tokio-stream.workspace = true
|
|
tokio-util.workspace = true
|
|
toml_edit = { workspace = true, features = [ "serde" ] }
|
|
tracing.workspace = true
|
|
twox-hash.workspace = true
|
|
url.workspace = true
|
|
walkdir.workspace = true
|
|
metrics.workspace = true
|
|
pageserver_api.workspace = true
|
|
pageserver_compaction.workspace = true
|
|
postgres_connection.workspace = true
|
|
postgres_ffi.workspace = true
|
|
pq_proto.workspace = true
|
|
remote_storage.workspace = true
|
|
storage_broker.workspace = true
|
|
tenant_size_model.workspace = true
|
|
utils.workspace = true
|
|
workspace_hack.workspace = true
|
|
reqwest.workspace = true
|
|
rpds.workspace = true
|
|
enum-map.workspace = true
|
|
enumset = { workspace = true, features = ["serde"]}
|
|
strum.workspace = true
|
|
strum_macros.workspace = true
|
|
|
|
[target.'cfg(target_os = "linux")'.dependencies]
|
|
procfs.workspace = true
|
|
|
|
[dev-dependencies]
|
|
criterion.workspace = true
|
|
hex-literal.workspace = true
|
|
tokio = { workspace = true, features = ["process", "sync", "fs", "rt", "io-util", "time", "test-util"] }
|
|
|
|
[[bench]]
|
|
name = "bench_layer_map"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "bench_walredo"
|
|
harness = false
|