mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-08 22:12:56 +00:00
4be2223a4c
This PR contains the first version of a [FoundationDB-like](https://www.youtube.com/watch?v=4fFDFbi3toc) simulation testing for safekeeper and walproposer. ### desim This is a core "framework" for running determenistic simulation. It operates on threads, allowing to test syncronous code (like walproposer). `libs/desim/src/executor.rs` contains implementation of a determenistic thread execution. This is achieved by blocking all threads, and each time allowing only a single thread to make an execution step. All executor's threads are blocked using `yield_me(after_ms)` function. This function is called when a thread wants to sleep or wait for an external notification (like blocking on a channel until it has a ready message). `libs/desim/src/chan.rs` contains implementation of a channel (basic sync primitive). It has unlimited capacity and any thread can push or read messages to/from it. `libs/desim/src/network.rs` has a very naive implementation of a network (only reliable TCP-like connections are supported for now), that can have arbitrary delays for each package and failure injections for breaking connections with some probability. `libs/desim/src/world.rs` ties everything together, to have a concept of virtual nodes that can have network connections between them. ### walproposer_sim Has everything to run walproposer and safekeepers in a simulation. `safekeeper.rs` reimplements all necesary stuff from `receive_wal.rs`, `send_wal.rs` and `timelines_global_map.rs`. `walproposer_api.rs` implements all walproposer callback to use simulation library. `simulation.rs` defines a schedule – a set of events like `restart <sk>` or `write_wal` that should happen at time `<ts>`. It also has code to spawn walproposer/safekeeper threads and provide config to them. ### tests `simple_test.rs` has tests that just start walproposer and 3 safekeepers together in a simulation, and tests that they are not crashing right away. `misc_test.rs` has tests checking more advanced simulation cases, like crashing or restarting threads, testing memory deallocation, etc. `random_test.rs` is the main test, it checks thousands of random seeds (schedules) for correctness. It roughly corresponds to running a real python integration test in an environment with very unstable network and cpu, but in a determenistic way (each seed results in the same execution log) and much much faster. Closes #547 --------- Co-authored-by: Arseny Sher <sher-ars@yandex.ru>
71 lines
1.9 KiB
TOML
71 lines
1.9 KiB
TOML
[package]
|
|
name = "safekeeper"
|
|
version = "0.1.0"
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
|
|
[features]
|
|
default = []
|
|
# Enables test-only APIs, incuding failpoints. In particular, enables the `fail_point!` macro,
|
|
# which adds some runtime cost to run tests on outage conditions
|
|
testing = ["fail/failpoints"]
|
|
|
|
[dependencies]
|
|
async-stream.workspace = true
|
|
anyhow.workspace = true
|
|
async-trait.workspace = true
|
|
byteorder.workspace = true
|
|
bytes.workspace = true
|
|
camino.workspace = true
|
|
camino-tempfile.workspace = true
|
|
chrono.workspace = true
|
|
clap = { workspace = true, features = ["derive"] }
|
|
const_format.workspace = true
|
|
crc32c.workspace = true
|
|
fail.workspace = true
|
|
fs2.workspace = true
|
|
git-version.workspace = true
|
|
hex.workspace = true
|
|
humantime.workspace = true
|
|
hyper.workspace = true
|
|
futures.workspace = true
|
|
once_cell.workspace = true
|
|
parking_lot.workspace = true
|
|
postgres.workspace = true
|
|
postgres-protocol.workspace = true
|
|
regex.workspace = true
|
|
scopeguard.workspace = true
|
|
reqwest = { workspace = true, features = ["json"] }
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
serde_with.workspace = true
|
|
signal-hook.workspace = true
|
|
thiserror.workspace = true
|
|
tokio = { workspace = true, features = ["fs"] }
|
|
tokio-util = { workspace = true }
|
|
tokio-io-timeout.workspace = true
|
|
tokio-postgres.workspace = true
|
|
toml_edit.workspace = true
|
|
tracing.workspace = true
|
|
url.workspace = true
|
|
metrics.workspace = true
|
|
postgres_backend.workspace = true
|
|
postgres_ffi.workspace = true
|
|
pq_proto.workspace = true
|
|
remote_storage.workspace = true
|
|
safekeeper_api.workspace = true
|
|
sha2.workspace = true
|
|
sd-notify.workspace = true
|
|
storage_broker.workspace = true
|
|
tokio-stream.workspace = true
|
|
utils.workspace = true
|
|
|
|
workspace_hack.workspace = true
|
|
|
|
[dev-dependencies]
|
|
walproposer.workspace = true
|
|
rand.workspace = true
|
|
desim.workspace = true
|
|
tracing.workspace = true
|
|
tracing-subscriber = { workspace = true, features = ["json"] }
|