mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-04 12:02:55 +00:00
f3073a4db9
Replace the layer array and linear search with R-tree So far, the in-memory layer map that holds information about layer files that exist, has used a simple Vec, in no particular order, to hold information about all the layers. That obviously doesn't scale very well; with thousands of layer files the linear search was consuming a lot of CPU. Replace it with a two-dimensional R-tree, with Key and LSN ranges as the dimensions. For the R-tree, use the 'rstar' crate. To be able to use that, we convert the Keys and LSNs into 256-bit integers. 64 bits would be enough to represent LSNs, and 128 bits would be enough to represent Keys. However, we use 256 bits, because rstar internally performs multiplication to calculate the area of rectangles, and the result of multiplying two 128 bit integers doesn't necessarily fit in 128 bits, causing integer overflow and, if overflow-checks are enabled, panic. To avoid that, we use 256 bit integers. Add a performance test that creates a lot of layer files, to demonstrate the benefit.
73 lines
2.2 KiB
TOML
73 lines
2.2 KiB
TOML
[package]
|
|
name = "pageserver"
|
|
version = "0.1.0"
|
|
edition = "2021"
|
|
|
|
[features]
|
|
default = []
|
|
# Enables test-only APIs, incuding failpoints. In particular, enables the `fail_point!` macro,
|
|
# which adds some runtime cost to run tests on outage conditions
|
|
testing = ["fail/failpoints"]
|
|
|
|
profiling = ["pprof"]
|
|
|
|
[dependencies]
|
|
async-stream = "0.3"
|
|
async-trait = "0.1"
|
|
chrono = "0.4.19"
|
|
rand = "0.8.3"
|
|
regex = "1.4.5"
|
|
bytes = "1.0.1"
|
|
byteorder = "1.4.3"
|
|
futures = "0.3.13"
|
|
hex = "0.4.3"
|
|
hyper = "0.14"
|
|
itertools = "0.10.3"
|
|
clap = "3.0"
|
|
daemonize = "0.4.1"
|
|
tokio = { version = "1.17", features = ["process", "sync", "macros", "fs", "rt", "io-util", "time"] }
|
|
tokio-util = { version = "0.7.3", features = ["io", "io-util"] }
|
|
postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
|
|
postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
|
|
postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
|
|
tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
|
|
anyhow = { version = "1.0", features = ["backtrace"] }
|
|
crc32c = "0.6.0"
|
|
thiserror = "1.0"
|
|
tar = "0.4.33"
|
|
humantime = "2.1.0"
|
|
serde = { version = "1.0", features = ["derive"] }
|
|
serde_json = "1"
|
|
serde_with = "1.12.0"
|
|
humantime-serde = "1.1.1"
|
|
|
|
pprof = { git = "https://github.com/neondatabase/pprof-rs.git", branch = "wallclock-profiling", features = ["flamegraph"], optional = true }
|
|
|
|
toml_edit = { version = "0.13", features = ["easy"] }
|
|
scopeguard = "1.1.0"
|
|
const_format = "0.2.21"
|
|
tracing = "0.1.36"
|
|
signal-hook = "0.3.10"
|
|
url = "2"
|
|
nix = "0.23"
|
|
once_cell = "1.13.0"
|
|
crossbeam-utils = "0.8.5"
|
|
fail = "0.5.0"
|
|
git-version = "0.3.5"
|
|
rstar = "0.9.3"
|
|
num-traits = "0.2.15"
|
|
amplify_num = "0.4.1"
|
|
|
|
postgres_ffi = { path = "../libs/postgres_ffi" }
|
|
etcd_broker = { path = "../libs/etcd_broker" }
|
|
metrics = { path = "../libs/metrics" }
|
|
utils = { path = "../libs/utils" }
|
|
remote_storage = { path = "../libs/remote_storage" }
|
|
workspace_hack = { version = "0.1", path = "../workspace_hack" }
|
|
close_fds = "0.3.2"
|
|
walkdir = "2.3.2"
|
|
|
|
[dev-dependencies]
|
|
hex-literal = "0.3"
|
|
tempfile = "3.2"
|