mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-08 22:12:56 +00:00
dbebede7bf
## Problem
Safekeepers currently decode and interpret WAL for each shard
separately.
This is wasteful in terms of CPU memory usage - we've seen this in
profiles.
## Summary of changes
Fan-out interpreted WAL to multiple shards.
The basic is that wal decoding and interpretation happens in a separate
tokio task and senders
attach to it. Senders only receive batches concerning their shard and
only past the Lsn they've last seen.
Fan-out is gated behind the `wal_reader_fanout` safekeeper flag
(disabled by default for now).
When fan-out is enabled, it might be desirable to control the absolute
delta between the
current position and a new shard's desired position (i.e. how far behind
or ahead a shard may be).
`max_delta_for_fanout` is a new optional safekeeper flag which dictates
whether to create a new
WAL reader or attach to the existing one. By default, this behaviour is
disabled. Let's consider enabling
it if we spot the need for it in the field.
## Testing
Tests passed [here](https://github.com/neondatabase/neon/pull/10301)
with wal reader fanout enabled
as of
34f6a71718.
Related: https://github.com/neondatabase/neon/issues/9337
Epic: https://github.com/neondatabase/neon/issues/9329
85 lines
2.2 KiB
TOML
85 lines
2.2 KiB
TOML
[package]
|
|
name = "safekeeper"
|
|
version = "0.1.0"
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
|
|
[features]
|
|
default = []
|
|
# Enables test-only APIs, incuding failpoints. In particular, enables the `fail_point!` macro,
|
|
# which adds some runtime cost to run tests on outage conditions
|
|
testing = ["fail/failpoints"]
|
|
benchmarking = []
|
|
|
|
[dependencies]
|
|
async-stream.workspace = true
|
|
anyhow.workspace = true
|
|
byteorder.workspace = true
|
|
bytes.workspace = true
|
|
camino.workspace = true
|
|
camino-tempfile.workspace = true
|
|
chrono.workspace = true
|
|
clap = { workspace = true, features = ["derive"] }
|
|
crc32c.workspace = true
|
|
fail.workspace = true
|
|
hex.workspace = true
|
|
humantime.workspace = true
|
|
http.workspace = true
|
|
hyper0.workspace = true
|
|
itertools.workspace = true
|
|
futures.workspace = true
|
|
once_cell.workspace = true
|
|
parking_lot.workspace = true
|
|
pageserver_api.workspace = true
|
|
postgres.workspace = true
|
|
postgres-protocol.workspace = true
|
|
pprof.workspace = true
|
|
rand.workspace = true
|
|
regex.workspace = true
|
|
scopeguard.workspace = true
|
|
reqwest = { workspace = true, features = ["json"] }
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
smallvec.workspace = true
|
|
strum.workspace = true
|
|
strum_macros.workspace = true
|
|
thiserror.workspace = true
|
|
tikv-jemallocator.workspace = true
|
|
tokio = { workspace = true, features = ["fs"] }
|
|
tokio-util = { workspace = true }
|
|
tokio-io-timeout.workspace = true
|
|
tokio-postgres.workspace = true
|
|
tokio-tar.workspace = true
|
|
tracing.workspace = true
|
|
url.workspace = true
|
|
metrics.workspace = true
|
|
postgres_backend.workspace = true
|
|
postgres_ffi.workspace = true
|
|
pq_proto.workspace = true
|
|
remote_storage.workspace = true
|
|
safekeeper_api.workspace = true
|
|
safekeeper_client.workspace = true
|
|
sha2.workspace = true
|
|
sd-notify.workspace = true
|
|
storage_broker.workspace = true
|
|
tokio-stream.workspace = true
|
|
utils.workspace = true
|
|
wal_decoder.workspace = true
|
|
env_logger.workspace = true
|
|
|
|
workspace_hack.workspace = true
|
|
|
|
[dev-dependencies]
|
|
criterion.workspace = true
|
|
itertools.workspace = true
|
|
walproposer.workspace = true
|
|
rand.workspace = true
|
|
desim.workspace = true
|
|
tracing.workspace = true
|
|
tracing-subscriber = { workspace = true, features = ["json"] }
|
|
|
|
[[bench]]
|
|
name = "receive_wal"
|
|
harness = false
|
|
required-features = ["benchmarking"]
|