mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-14 00:42:54 +00:00
045bc6af8b
Rebased version of #5234, part of #6768 This consists of three parts: 1. A refactoring and new contract for implementing and testing compaction. The logic is now in a separate crate, with no dependency on the 'pageserver' crate. It defines an interface that the real pageserver must implement, in order to call the compaction algorithm. The interface models things like delta and image layers, but just the parts that the compaction algorithm needs to make decisions. That makes it easier unit test the algorithm and experiment with different implementations. I did not convert the current code to the new abstraction, however. When compaction algorithm is set to "Legacy", we just use the old code. It might be worthwhile to convert the old code to the new abstraction, so that we can compare the behavior of the new algorithm against the old one, using the same simulated cases. If we do that, have to be careful that the converted code really is equivalent to the old. This inclues only trivial changes to the main pageserver code. All the new code is behind a tenant config option. So this should be pretty safe to merge, even if the new implementation is buggy, as long as we don't enable it. 2. A new compaction algorithm, implemented using the new abstraction. The new algorithm is tiered compaction. It is inspired by the PoC at PR #4539, although I did not use that code directly, as I needed the new implementation to fit the new abstraction. The algorithm here is less advanced, I did not implement partial image layers, for example. I wanted to keep it simple on purpose, so that as we add bells and whistles, we can see the effects using the included simulator. One difference to #4539 and your typical LSM tree implementations is how we keep track of the LSM tree levels. This PR doesn't have a permanent concept of a level, tier or sorted run at all. There are just delta and image layers. However, when compaction starts, we look at the layers that exist, and arrange them into levels, depending on their shapes. That is ephemeral: when the compaction finishes, we forget that information. This allows the new algorithm to work without any extra bookkeeping. That makes it easier to transition from the old algorithm to new, and back again. There is just a new tenant config option to choose the compaction algorithm. The default is "Legacy", meaning the current algorithm in 'main'. If you set it to "Tiered", the new algorithm is used. 3. A simulator, which implements the new abstraction. The simulator can be used to analyze write and storage amplification, without running a test with the full pageserver. It can also draw an SVG animation of the simulation, to visualize how layers are created and deleted. To run the simulator: cargo run --bin compaction-simulator run-suite --------- Co-authored-by: Heikki Linnakangas <heikki@neon.tech>
104 lines
3.0 KiB
TOML
104 lines
3.0 KiB
TOML
[package]
|
|
name = "pageserver"
|
|
version = "0.1.0"
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
|
|
[features]
|
|
default = []
|
|
# Enables test-only APIs, incuding failpoints. In particular, enables the `fail_point!` macro,
|
|
# which adds some runtime cost to run tests on outage conditions
|
|
testing = ["fail/failpoints"]
|
|
|
|
[dependencies]
|
|
anyhow.workspace = true
|
|
arc-swap.workspace = true
|
|
async-compression.workspace = true
|
|
async-stream.workspace = true
|
|
async-trait.workspace = true
|
|
byteorder.workspace = true
|
|
bytes.workspace = true
|
|
camino.workspace = true
|
|
camino-tempfile.workspace = true
|
|
chrono = { workspace = true, features = ["serde"] }
|
|
clap = { workspace = true, features = ["string"] }
|
|
const_format.workspace = true
|
|
consumption_metrics.workspace = true
|
|
crc32c.workspace = true
|
|
crossbeam-utils.workspace = true
|
|
either.workspace = true
|
|
flate2.workspace = true
|
|
fail.workspace = true
|
|
futures.workspace = true
|
|
git-version.workspace = true
|
|
hex.workspace = true
|
|
humantime.workspace = true
|
|
humantime-serde.workspace = true
|
|
hyper.workspace = true
|
|
itertools.workspace = true
|
|
leaky-bucket.workspace = true
|
|
md5.workspace = true
|
|
nix.workspace = true
|
|
# hack to get the number of worker threads tokio uses
|
|
num_cpus = { version = "1.15" }
|
|
num-traits.workspace = true
|
|
once_cell.workspace = true
|
|
pin-project-lite.workspace = true
|
|
postgres.workspace = true
|
|
postgres_backend.workspace = true
|
|
postgres-protocol.workspace = true
|
|
postgres-types.workspace = true
|
|
rand.workspace = true
|
|
regex.workspace = true
|
|
scopeguard.workspace = true
|
|
serde.workspace = true
|
|
serde_json = { workspace = true, features = ["raw_value"] }
|
|
serde_path_to_error.workspace = true
|
|
serde_with.workspace = true
|
|
signal-hook.workspace = true
|
|
smallvec = { workspace = true, features = ["write"] }
|
|
svg_fmt.workspace = true
|
|
sync_wrapper.workspace = true
|
|
tokio-tar.workspace = true
|
|
thiserror.workspace = true
|
|
tokio = { workspace = true, features = ["process", "sync", "fs", "rt", "io-util", "time"] }
|
|
tokio-epoll-uring.workspace = true
|
|
tokio-io-timeout.workspace = true
|
|
tokio-postgres.workspace = true
|
|
tokio-stream.workspace = true
|
|
tokio-util.workspace = true
|
|
toml_edit = { workspace = true, features = [ "serde" ] }
|
|
tracing.workspace = true
|
|
url.workspace = true
|
|
walkdir.workspace = true
|
|
metrics.workspace = true
|
|
pageserver_api.workspace = true
|
|
pageserver_compaction.workspace = true
|
|
postgres_connection.workspace = true
|
|
postgres_ffi.workspace = true
|
|
pq_proto.workspace = true
|
|
remote_storage.workspace = true
|
|
storage_broker.workspace = true
|
|
tenant_size_model.workspace = true
|
|
utils.workspace = true
|
|
workspace_hack.workspace = true
|
|
reqwest.workspace = true
|
|
rpds.workspace = true
|
|
enum-map.workspace = true
|
|
enumset = { workspace = true, features = ["serde"]}
|
|
strum.workspace = true
|
|
strum_macros.workspace = true
|
|
|
|
[dev-dependencies]
|
|
criterion.workspace = true
|
|
hex-literal.workspace = true
|
|
tokio = { workspace = true, features = ["process", "sync", "fs", "rt", "io-util", "time", "test-util"] }
|
|
|
|
[[bench]]
|
|
name = "bench_layer_map"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "bench_walredo"
|
|
harness = false
|