mirror of
https://github.com/neondatabase/neon.git
synced 2025-12-22 21:59:59 +00:00
8223c1ba9d
## Problem The communicator will need gRPC channel/client/stream pools for efficient reuse across many backends. Touches #11735. Requires #12396. ## Summary of changes Adds three nested resource pools: * `ChannelPool` for gRPC channels (i.e. TCP connections). * `ClientPool` for gRPC clients (i.e. `page_api::Client`). Acquires channels from `ChannelPool`. * `StreamPool` for gRPC GetPage streams. Acquires clients from `ClientPool`. These are minimal functional implementations that will need further improvements and performance optimization. However, the overall structure is expected to be roughly final, so reviews should focus on that. The pools are not yet in use, but will form the foundation of a rich gRPC Pageserver client used by the communicator (see #12462). This PR also adds the initial crate scaffolding for that client. See doc comments for details.
367 lines
13 KiB
TOML
367 lines
13 KiB
TOML
[workspace]
|
|
resolver = "2"
|
|
members = [
|
|
"compute_tools",
|
|
"control_plane",
|
|
"control_plane/storcon_cli",
|
|
"pageserver",
|
|
"pageserver/compaction",
|
|
"pageserver/ctl",
|
|
"pageserver/client",
|
|
"pageserver/client_grpc",
|
|
"pageserver/pagebench",
|
|
"pageserver/page_api",
|
|
"proxy",
|
|
"safekeeper",
|
|
"safekeeper/client",
|
|
"storage_broker",
|
|
"storage_controller",
|
|
"storage_controller/client",
|
|
"storage_scrubber",
|
|
"workspace_hack",
|
|
"libs/compute_api",
|
|
"libs/http-utils",
|
|
"libs/pageserver_api",
|
|
"libs/postgres_ffi",
|
|
"libs/postgres_ffi_types",
|
|
"libs/postgres_versioninfo",
|
|
"libs/safekeeper_api",
|
|
"libs/desim",
|
|
"libs/neon-shmem",
|
|
"libs/utils",
|
|
"libs/consumption_metrics",
|
|
"libs/postgres_backend",
|
|
"libs/posthog_client_lite",
|
|
"libs/pq_proto",
|
|
"libs/tenant_size_model",
|
|
"libs/metrics",
|
|
"libs/postgres_connection",
|
|
"libs/remote_storage",
|
|
"libs/tracing-utils",
|
|
"libs/postgres_ffi/wal_craft",
|
|
"libs/vm_monitor",
|
|
"libs/walproposer",
|
|
"libs/wal_decoder",
|
|
"libs/postgres_initdb",
|
|
"libs/proxy/json",
|
|
"libs/proxy/postgres-protocol2",
|
|
"libs/proxy/postgres-types2",
|
|
"libs/proxy/tokio-postgres2",
|
|
"endpoint_storage",
|
|
"pgxn/neon/communicator",
|
|
]
|
|
|
|
[workspace.package]
|
|
edition = "2024"
|
|
license = "Apache-2.0"
|
|
|
|
## All dependency versions, used in the project
|
|
[workspace.dependencies]
|
|
ahash = "0.8"
|
|
anyhow = { version = "1.0", features = ["backtrace"] }
|
|
arc-swap = "1.7"
|
|
async-compression = { version = "0.4.0", features = ["tokio", "gzip", "zstd"] }
|
|
atomic-take = "1.1.0"
|
|
flate2 = "1.0.26"
|
|
assert-json-diff = "2"
|
|
async-stream = "0.3"
|
|
async-trait = "0.1"
|
|
aws-config = { version = "1.5", default-features = false, features=["rustls", "sso"] }
|
|
aws-sdk-s3 = "1.52"
|
|
aws-sdk-iam = "1.46.0"
|
|
aws-sdk-kms = "1.47.0"
|
|
aws-smithy-async = { version = "1.2.1", default-features = false, features=["rt-tokio"] }
|
|
aws-smithy-types = "1.2"
|
|
aws-credential-types = "1.2.0"
|
|
aws-sigv4 = { version = "1.2", features = ["sign-http"] }
|
|
aws-types = "1.3"
|
|
axum = { version = "0.8.1", features = ["ws"] }
|
|
axum-extra = { version = "0.10.0", features = ["typed-header", "query"] }
|
|
base64 = "0.22"
|
|
bincode = "1.3"
|
|
bindgen = "0.71"
|
|
bit_field = "0.10.2"
|
|
bstr = "1.0"
|
|
byteorder = "1.4"
|
|
bytes = "1.9"
|
|
camino = "1.1.6"
|
|
cfg-if = "1.0.0"
|
|
cron = "0.15"
|
|
chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
|
clap = { version = "4.0", features = ["derive", "env"] }
|
|
clashmap = { version = "1.0", features = ["raw-api"] }
|
|
comfy-table = "7.1"
|
|
const_format = "0.2"
|
|
crc32c = "0.6"
|
|
diatomic-waker = { version = "0.2.3" }
|
|
either = "1.8"
|
|
enum-map = "2.4.2"
|
|
enumset = "1.0.12"
|
|
fail = "0.5.0"
|
|
fallible-iterator = "0.2"
|
|
framed-websockets = { version = "0.1.0", git = "https://github.com/neondatabase/framed-websockets" }
|
|
futures = "0.3"
|
|
futures-core = "0.3"
|
|
futures-util = "0.3"
|
|
git-version = "0.3"
|
|
governor = "0.8"
|
|
hashbrown = "0.14"
|
|
hashlink = "0.9.1"
|
|
hdrhistogram = "7.5.2"
|
|
hex = "0.4"
|
|
hex-literal = "0.4"
|
|
hmac = "0.12.1"
|
|
hostname = "0.4"
|
|
http = {version = "1.1.0", features = ["std"]}
|
|
http-types = { version = "2", default-features = false }
|
|
http-body-util = "0.1.2"
|
|
humantime = "2.2"
|
|
humantime-serde = "1.1.1"
|
|
hyper0 = { package = "hyper", version = "0.14" }
|
|
hyper = "1.4"
|
|
hyper-util = "0.1"
|
|
tokio-tungstenite = "0.21.0"
|
|
indexmap = { version = "2", features = ["serde"] }
|
|
indoc = "2"
|
|
ipnet = "2.10.0"
|
|
itertools = "0.10"
|
|
itoa = "1.0.11"
|
|
jemalloc_pprof = { version = "0.7", features = ["symbolize", "flamegraph"] }
|
|
jsonwebtoken = "9"
|
|
lasso = "0.7"
|
|
libc = "0.2"
|
|
md5 = "0.7.0"
|
|
measured = { version = "0.0.22", features=["lasso"] }
|
|
measured-process = { version = "0.0.22" }
|
|
memoffset = "0.9"
|
|
nix = { version = "0.30.1", features = ["dir", "fs", "mman", "process", "socket", "signal", "poll"] }
|
|
# Do not update to >= 7.0.0, at least. The update will have a significant impact
|
|
# on compute startup metrics (start_postgres_ms), >= 25% degradation.
|
|
notify = "6.0.0"
|
|
num_cpus = "1.15"
|
|
num-traits = "0.2.19"
|
|
once_cell = "1.13"
|
|
opentelemetry = "0.27"
|
|
opentelemetry_sdk = "0.27"
|
|
opentelemetry-otlp = { version = "0.27", default-features = false, features = ["http-proto", "trace", "http", "reqwest-client"] }
|
|
opentelemetry-semantic-conventions = "0.27"
|
|
parking_lot = "0.12"
|
|
parquet = { version = "53", default-features = false, features = ["zstd"] }
|
|
parquet_derive = "53"
|
|
pbkdf2 = { version = "0.12.1", features = ["simple", "std"] }
|
|
pem = "3.0.3"
|
|
pin-project-lite = "0.2"
|
|
pprof = { version = "0.14", features = ["criterion", "flamegraph", "frame-pointer", "prost-codec"] }
|
|
procfs = "0.16"
|
|
prometheus = {version = "0.13", default-features=false, features = ["process"]} # removes protobuf dependency
|
|
prost = "0.13.5"
|
|
prost-types = "0.13.5"
|
|
rand = "0.8"
|
|
redis = { version = "0.29.2", features = ["tokio-rustls-comp", "keep-alive"] }
|
|
regex = "1.10.2"
|
|
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
|
|
reqwest-tracing = { version = "0.5", features = ["opentelemetry_0_27"] }
|
|
reqwest-middleware = "0.4"
|
|
reqwest-retry = "0.7"
|
|
routerify = "3"
|
|
rpds = "0.13"
|
|
rustc-hash = "1.1.0"
|
|
rustls = { version = "0.23.16", default-features = false }
|
|
rustls-pemfile = "2"
|
|
rustls-pki-types = "1.11"
|
|
scopeguard = "1.1"
|
|
sysinfo = "0.29.2"
|
|
sd-notify = "0.4.1"
|
|
send-future = "0.1.0"
|
|
sentry = { version = "0.37", default-features = false, features = ["backtrace", "contexts", "panic", "rustls", "reqwest" ] }
|
|
serde = { version = "1.0", features = ["derive"] }
|
|
serde_json = "1"
|
|
serde_path_to_error = "0.1"
|
|
serde_with = { version = "3", features = [ "base64" ] }
|
|
serde_assert = "0.5.0"
|
|
serde_repr = "0.1.20"
|
|
sha2 = "0.10.2"
|
|
signal-hook = "0.3"
|
|
smallvec = "1.11"
|
|
smol_str = { version = "0.2.0", features = ["serde"] }
|
|
socket2 = "0.5"
|
|
spki = "0.7.3"
|
|
strum = "0.26"
|
|
strum_macros = "0.26"
|
|
"subtle" = "2.5.0"
|
|
svg_fmt = "0.4.3"
|
|
sync_wrapper = "0.1.2"
|
|
tar = "0.4"
|
|
test-context = "0.3"
|
|
thiserror = "1.0"
|
|
tikv-jemallocator = { version = "0.6", features = ["profiling", "stats", "unprefixed_malloc_on_supported_platforms"] }
|
|
tikv-jemalloc-ctl = { version = "0.6", features = ["stats"] }
|
|
tokio = { version = "1.43.1", features = ["macros"] }
|
|
tokio-epoll-uring = { git = "https://github.com/neondatabase/tokio-epoll-uring.git" , branch = "main" }
|
|
tokio-io-timeout = "1.2.0"
|
|
tokio-postgres-rustls = "0.12.0"
|
|
tokio-rustls = { version = "0.26.0", default-features = false, features = ["tls12", "ring"]}
|
|
tokio-stream = "0.1"
|
|
tokio-tar = "0.3"
|
|
tokio-util = { version = "0.7.10", features = ["io", "io-util", "rt"] }
|
|
toml = "0.8"
|
|
toml_edit = "0.22"
|
|
tonic = { version = "0.13.1", default-features = false, features = ["channel", "codegen", "gzip", "prost", "router", "server", "tls-ring", "tls-native-roots", "zstd"] }
|
|
tonic-reflection = { version = "0.13.1", features = ["server"] }
|
|
tower = { version = "0.5.2", default-features = false }
|
|
tower-http = { version = "0.6.2", features = ["auth", "request-id", "trace"] }
|
|
|
|
# This revision uses opentelemetry 0.27. There's no tag for it.
|
|
tower-otel = { git = "https://github.com/mattiapenati/tower-otel", rev = "56a7321053bcb72443888257b622ba0d43a11fcd" }
|
|
|
|
tower-service = "0.3.3"
|
|
tracing = "0.1"
|
|
tracing-error = "0.2"
|
|
tracing-log = "0.2"
|
|
tracing-opentelemetry = "0.28"
|
|
tracing-serde = "0.2.0"
|
|
tracing-subscriber = { version = "0.3", default-features = false, features = ["smallvec", "fmt", "tracing-log", "std", "env-filter", "json"] }
|
|
try-lock = "0.2.5"
|
|
test-log = { version = "0.2.17", default-features = false, features = ["log"] }
|
|
twox-hash = { version = "1.6.3", default-features = false }
|
|
typed-json = "0.1"
|
|
url = "2.2"
|
|
urlencoding = "2.1"
|
|
uuid = { version = "1.6.1", features = ["v4", "v7", "serde"] }
|
|
walkdir = "2.3.2"
|
|
rustls-native-certs = "0.8"
|
|
whoami = "1.5.1"
|
|
zerocopy = { version = "0.8", features = ["derive", "simd"] }
|
|
json-structural-diff = { version = "0.2.0" }
|
|
x509-cert = { version = "0.2.5" }
|
|
|
|
## TODO replace this with tracing
|
|
env_logger = "0.11"
|
|
log = "0.4"
|
|
|
|
## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed
|
|
postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
|
|
postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
|
|
postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
|
|
tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
|
|
|
|
## Azure SDK crates
|
|
azure_core = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] }
|
|
azure_identity = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] }
|
|
azure_storage = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] }
|
|
azure_storage_blobs = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] }
|
|
|
|
## Local libraries
|
|
compute_api = { version = "0.1", path = "./libs/compute_api/" }
|
|
consumption_metrics = { version = "0.1", path = "./libs/consumption_metrics/" }
|
|
desim = { version = "0.1", path = "./libs/desim" }
|
|
endpoint_storage = { version = "0.0.1", path = "./endpoint_storage/" }
|
|
http-utils = { version = "0.1", path = "./libs/http-utils/" }
|
|
metrics = { version = "0.1", path = "./libs/metrics/" }
|
|
neon-shmem = { version = "0.1", path = "./libs/neon-shmem/" }
|
|
pageserver = { path = "./pageserver" }
|
|
pageserver_api = { version = "0.1", path = "./libs/pageserver_api/" }
|
|
pageserver_client = { path = "./pageserver/client" }
|
|
pageserver_compaction = { version = "0.1", path = "./pageserver/compaction/" }
|
|
pageserver_page_api = { path = "./pageserver/page_api" }
|
|
postgres_backend = { version = "0.1", path = "./libs/postgres_backend/" }
|
|
postgres_connection = { version = "0.1", path = "./libs/postgres_connection/" }
|
|
postgres_ffi = { version = "0.1", path = "./libs/postgres_ffi/" }
|
|
postgres_ffi_types = { version = "0.1", path = "./libs/postgres_ffi_types/" }
|
|
postgres_versioninfo = { version = "0.1", path = "./libs/postgres_versioninfo/" }
|
|
postgres_initdb = { path = "./libs/postgres_initdb" }
|
|
posthog_client_lite = { version = "0.1", path = "./libs/posthog_client_lite" }
|
|
pq_proto = { version = "0.1", path = "./libs/pq_proto/" }
|
|
remote_storage = { version = "0.1", path = "./libs/remote_storage/" }
|
|
safekeeper_api = { version = "0.1", path = "./libs/safekeeper_api" }
|
|
safekeeper_client = { path = "./safekeeper/client" }
|
|
storage_broker = { version = "0.1", path = "./storage_broker/" } # Note: main broker code is inside the binary crate, so linking with the library shouldn't be heavy.
|
|
storage_controller_client = { path = "./storage_controller/client" }
|
|
tenant_size_model = { version = "0.1", path = "./libs/tenant_size_model/" }
|
|
tracing-utils = { version = "0.1", path = "./libs/tracing-utils/" }
|
|
utils = { version = "0.1", path = "./libs/utils/" }
|
|
vm_monitor = { version = "0.1", path = "./libs/vm_monitor/" }
|
|
wal_decoder = { version = "0.1", path = "./libs/wal_decoder" }
|
|
walproposer = { version = "0.1", path = "./libs/walproposer/" }
|
|
|
|
## Common library dependency
|
|
workspace_hack = { version = "0.1", path = "./workspace_hack/" }
|
|
|
|
## Build dependencies
|
|
cbindgen = "0.29.0"
|
|
criterion = "0.5.1"
|
|
rcgen = "0.13"
|
|
rstest = "0.18"
|
|
camino-tempfile = "1.0.2"
|
|
tonic-build = "0.13.1"
|
|
|
|
[patch.crates-io]
|
|
|
|
# Needed to get `tokio-postgres-rustls` to depend on our fork.
|
|
tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
|
|
|
|
################# Binary contents sections
|
|
|
|
[profile.release]
|
|
# This is useful for profiling and, to some extent, debug.
|
|
# Besides, debug info should not affect the performance.
|
|
#
|
|
# NB: we also enable frame pointers for improved profiling, see .cargo/config.toml.
|
|
debug = true
|
|
|
|
# disable debug symbols for all packages except this one to decrease binaries size
|
|
[profile.release.package."*"]
|
|
debug = false
|
|
|
|
[profile.release-line-debug]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
[profile.release-line-debug-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
lto = true
|
|
|
|
[profile.release-line-debug-size]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
[profile.release-line-debug-zize]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
[profile.release-line-debug-size-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
lto = true
|
|
[profile.release-line-debug-zize-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
lto = true
|
|
|
|
[profile.release-no-debug]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
|
|
[profile.release-no-debug-size]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
[profile.release-no-debug-zize]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
|
|
[profile.release-no-debug-size-lto]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
lto = true
|
|
|
|
[profile.release-no-debug-zize-lto]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
lto = true
|