mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-09 06:22:57 +00:00
dcb24ce170
## Problem We don't have good observability for memory usage. This would be useful e.g. to debug OOM incidents or optimize performance or resource usage. We would also like to use continuous profiling with e.g. [Grafana Cloud Profiles](https://grafana.com/products/cloud/profiles-for-continuous-profiling/) (see https://github.com/neondatabase/cloud/issues/14888). This PR is intended as a proof of concept, to try it out in staging and drive further discussions about profiling more broadly. Touches https://github.com/neondatabase/neon/issues/9534. Touches https://github.com/neondatabase/cloud/issues/14888. Depends on #9779. Depends on #9780. ## Summary of changes Adds a HTTP route `/profile/heap` that takes a heap profile and returns it. Query parameters: * `format`: output format (`jemalloc` or `pprof`; default `pprof`). Unlike CPU profiles (see #9764), heap profiles are not symbolized and require the original binary to translate addresses to function names. To make this work with Grafana, we'll probably have to symbolize the process server-side -- this is left as future work, as is other output formats like SVG. Heap profiles don't work on macOS due to limitations in jemalloc.
321 lines
11 KiB
TOML
321 lines
11 KiB
TOML
[workspace]
|
|
resolver = "2"
|
|
members = [
|
|
"compute_tools",
|
|
"control_plane",
|
|
"control_plane/storcon_cli",
|
|
"pageserver",
|
|
"pageserver/compaction",
|
|
"pageserver/ctl",
|
|
"pageserver/client",
|
|
"pageserver/pagebench",
|
|
"proxy",
|
|
"safekeeper",
|
|
"storage_broker",
|
|
"storage_controller",
|
|
"storage_controller/client",
|
|
"storage_scrubber",
|
|
"workspace_hack",
|
|
"libs/compute_api",
|
|
"libs/pageserver_api",
|
|
"libs/postgres_ffi",
|
|
"libs/safekeeper_api",
|
|
"libs/desim",
|
|
"libs/utils",
|
|
"libs/consumption_metrics",
|
|
"libs/postgres_backend",
|
|
"libs/pq_proto",
|
|
"libs/tenant_size_model",
|
|
"libs/metrics",
|
|
"libs/postgres_connection",
|
|
"libs/remote_storage",
|
|
"libs/tracing-utils",
|
|
"libs/postgres_ffi/wal_craft",
|
|
"libs/vm_monitor",
|
|
"libs/walproposer",
|
|
"libs/wal_decoder",
|
|
"libs/postgres_initdb",
|
|
"libs/proxy/postgres-protocol2",
|
|
"libs/proxy/postgres-types2",
|
|
"libs/proxy/tokio-postgres2",
|
|
]
|
|
|
|
[workspace.package]
|
|
edition = "2021"
|
|
license = "Apache-2.0"
|
|
|
|
## All dependency versions, used in the project
|
|
[workspace.dependencies]
|
|
ahash = "0.8"
|
|
anyhow = { version = "1.0", features = ["backtrace"] }
|
|
arc-swap = "1.6"
|
|
async-compression = { version = "0.4.0", features = ["tokio", "gzip", "zstd"] }
|
|
atomic-take = "1.1.0"
|
|
azure_core = { version = "0.19", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] }
|
|
azure_identity = { version = "0.19", default-features = false, features = ["enable_reqwest_rustls"] }
|
|
azure_storage = { version = "0.19", default-features = false, features = ["enable_reqwest_rustls"] }
|
|
azure_storage_blobs = { version = "0.19", default-features = false, features = ["enable_reqwest_rustls"] }
|
|
flate2 = "1.0.26"
|
|
async-stream = "0.3"
|
|
async-trait = "0.1"
|
|
aws-config = { version = "1.5", default-features = false, features=["rustls", "sso"] }
|
|
aws-sdk-s3 = "1.52"
|
|
aws-sdk-iam = "1.46.0"
|
|
aws-sdk-kms = "1.47.0"
|
|
aws-smithy-async = { version = "1.2.1", default-features = false, features=["rt-tokio"] }
|
|
aws-smithy-types = "1.2"
|
|
aws-credential-types = "1.2.0"
|
|
aws-sigv4 = { version = "1.2", features = ["sign-http"] }
|
|
aws-types = "1.3"
|
|
axum = { version = "0.7.5", features = ["ws"] }
|
|
base64 = "0.13.0"
|
|
bincode = "1.3"
|
|
bindgen = "0.70"
|
|
bit_field = "0.10.2"
|
|
bstr = "1.0"
|
|
byteorder = "1.4"
|
|
bytes = "1.0"
|
|
camino = "1.1.6"
|
|
cfg-if = "1.0.0"
|
|
chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
|
clap = { version = "4.0", features = ["derive", "env"] }
|
|
comfy-table = "7.1"
|
|
const_format = "0.2"
|
|
crc32c = "0.6"
|
|
dashmap = { version = "5.5.0", features = ["raw-api"] }
|
|
diatomic-waker = { version = "0.2.3" }
|
|
either = "1.8"
|
|
enum-map = "2.4.2"
|
|
enumset = "1.0.12"
|
|
fail = "0.5.0"
|
|
fallible-iterator = "0.2"
|
|
framed-websockets = { version = "0.1.0", git = "https://github.com/neondatabase/framed-websockets" }
|
|
futures = "0.3"
|
|
futures-core = "0.3"
|
|
futures-util = "0.3"
|
|
git-version = "0.3"
|
|
hashbrown = "0.14"
|
|
hashlink = "0.9.1"
|
|
hdrhistogram = "7.5.2"
|
|
hex = "0.4"
|
|
hex-literal = "0.4"
|
|
hmac = "0.12.1"
|
|
hostname = "0.4"
|
|
http = {version = "1.1.0", features = ["std"]}
|
|
http-types = { version = "2", default-features = false }
|
|
http-body-util = "0.1.2"
|
|
humantime = "2.1"
|
|
humantime-serde = "1.1.1"
|
|
hyper0 = { package = "hyper", version = "0.14" }
|
|
hyper = "1.4"
|
|
hyper-util = "0.1"
|
|
tokio-tungstenite = "0.21.0"
|
|
indexmap = "2"
|
|
indoc = "2"
|
|
ipnet = "2.10.0"
|
|
itertools = "0.10"
|
|
itoa = "1.0.11"
|
|
jemalloc_pprof = "0.6"
|
|
jsonwebtoken = "9"
|
|
lasso = "0.7"
|
|
libc = "0.2"
|
|
md5 = "0.7.0"
|
|
measured = { version = "0.0.22", features=["lasso"] }
|
|
measured-process = { version = "0.0.22" }
|
|
memoffset = "0.9"
|
|
nix = { version = "0.27", features = ["dir", "fs", "process", "socket", "signal", "poll"] }
|
|
notify = "6.0.0"
|
|
num_cpus = "1.15"
|
|
num-traits = "0.2.15"
|
|
once_cell = "1.13"
|
|
opentelemetry = "0.26"
|
|
opentelemetry_sdk = "0.26"
|
|
opentelemetry-otlp = { version = "0.26", default-features=false, features = ["http-proto", "trace", "http", "reqwest-client"] }
|
|
opentelemetry-semantic-conventions = "0.26"
|
|
parking_lot = "0.12"
|
|
parquet = { version = "53", default-features = false, features = ["zstd"] }
|
|
parquet_derive = "53"
|
|
pbkdf2 = { version = "0.12.1", features = ["simple", "std"] }
|
|
pin-project-lite = "0.2"
|
|
pprof = { version = "0.14", features = ["criterion", "flamegraph", "protobuf", "protobuf-codec"] }
|
|
procfs = "0.16"
|
|
prometheus = {version = "0.13", default-features=false, features = ["process"]} # removes protobuf dependency
|
|
prost = "0.13"
|
|
rand = "0.8"
|
|
redis = { version = "0.25.2", features = ["tokio-rustls-comp", "keep-alive"] }
|
|
regex = "1.10.2"
|
|
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
|
|
reqwest-tracing = { version = "0.5", features = ["opentelemetry_0_26"] }
|
|
reqwest-middleware = "0.4"
|
|
reqwest-retry = "0.7"
|
|
routerify = "3"
|
|
rpds = "0.13"
|
|
rustc-hash = "1.1.0"
|
|
rustls = { version = "0.23.16", default-features = false }
|
|
rustls-pemfile = "2"
|
|
scopeguard = "1.1"
|
|
sysinfo = "0.29.2"
|
|
sd-notify = "0.4.1"
|
|
send-future = "0.1.0"
|
|
sentry = { version = "0.32", default-features = false, features = ["backtrace", "contexts", "panic", "rustls", "reqwest" ] }
|
|
serde = { version = "1.0", features = ["derive"] }
|
|
serde_json = "1"
|
|
serde_path_to_error = "0.1"
|
|
serde_with = { version = "2.0", features = [ "base64" ] }
|
|
serde_assert = "0.5.0"
|
|
sha2 = "0.10.2"
|
|
signal-hook = "0.3"
|
|
smallvec = "1.11"
|
|
smol_str = { version = "0.2.0", features = ["serde"] }
|
|
socket2 = "0.5"
|
|
strum = "0.26"
|
|
strum_macros = "0.26"
|
|
"subtle" = "2.5.0"
|
|
svg_fmt = "0.4.3"
|
|
sync_wrapper = "0.1.2"
|
|
tar = "0.4"
|
|
test-context = "0.3"
|
|
thiserror = "1.0"
|
|
tikv-jemallocator = { version = "0.6", features = ["profiling", "stats", "unprefixed_malloc_on_supported_platforms"] }
|
|
tikv-jemalloc-ctl = { version = "0.6", features = ["stats"] }
|
|
tokio = { version = "1.17", features = ["macros"] }
|
|
tokio-epoll-uring = { git = "https://github.com/neondatabase/tokio-epoll-uring.git" , branch = "main" }
|
|
tokio-io-timeout = "1.2.0"
|
|
tokio-postgres-rustls = "0.12.0"
|
|
tokio-rustls = { version = "0.26.0", default-features = false, features = ["tls12", "ring"]}
|
|
tokio-stream = "0.1"
|
|
tokio-tar = "0.3"
|
|
tokio-util = { version = "0.7.10", features = ["io", "rt"] }
|
|
toml = "0.8"
|
|
toml_edit = "0.22"
|
|
tonic = {version = "0.12.3", features = ["tls", "tls-roots"]}
|
|
tower-service = "0.3.2"
|
|
tracing = "0.1"
|
|
tracing-error = "0.2"
|
|
tracing-opentelemetry = "0.27"
|
|
tracing-subscriber = { version = "0.3", default-features = false, features = ["smallvec", "fmt", "tracing-log", "std", "env-filter", "json"] }
|
|
try-lock = "0.2.5"
|
|
twox-hash = { version = "1.6.3", default-features = false }
|
|
typed-json = "0.1"
|
|
url = "2.2"
|
|
urlencoding = "2.1"
|
|
uuid = { version = "1.6.1", features = ["v4", "v7", "serde"] }
|
|
walkdir = "2.3.2"
|
|
rustls-native-certs = "0.8"
|
|
x509-parser = "0.16"
|
|
whoami = "1.5.1"
|
|
zerocopy = { version = "0.7", features = ["derive"] }
|
|
|
|
## TODO replace this with tracing
|
|
env_logger = "0.10"
|
|
log = "0.4"
|
|
|
|
## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed
|
|
postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
|
|
postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
|
|
postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
|
|
tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
|
|
|
|
## Local libraries
|
|
compute_api = { version = "0.1", path = "./libs/compute_api/" }
|
|
consumption_metrics = { version = "0.1", path = "./libs/consumption_metrics/" }
|
|
metrics = { version = "0.1", path = "./libs/metrics/" }
|
|
pageserver = { path = "./pageserver" }
|
|
pageserver_api = { version = "0.1", path = "./libs/pageserver_api/" }
|
|
pageserver_client = { path = "./pageserver/client" }
|
|
pageserver_compaction = { version = "0.1", path = "./pageserver/compaction/" }
|
|
postgres_backend = { version = "0.1", path = "./libs/postgres_backend/" }
|
|
postgres_connection = { version = "0.1", path = "./libs/postgres_connection/" }
|
|
postgres_ffi = { version = "0.1", path = "./libs/postgres_ffi/" }
|
|
postgres_initdb = { path = "./libs/postgres_initdb" }
|
|
pq_proto = { version = "0.1", path = "./libs/pq_proto/" }
|
|
remote_storage = { version = "0.1", path = "./libs/remote_storage/" }
|
|
safekeeper_api = { version = "0.1", path = "./libs/safekeeper_api" }
|
|
desim = { version = "0.1", path = "./libs/desim" }
|
|
storage_broker = { version = "0.1", path = "./storage_broker/" } # Note: main broker code is inside the binary crate, so linking with the library shouldn't be heavy.
|
|
storage_controller_client = { path = "./storage_controller/client" }
|
|
tenant_size_model = { version = "0.1", path = "./libs/tenant_size_model/" }
|
|
tracing-utils = { version = "0.1", path = "./libs/tracing-utils/" }
|
|
utils = { version = "0.1", path = "./libs/utils/" }
|
|
vm_monitor = { version = "0.1", path = "./libs/vm_monitor/" }
|
|
walproposer = { version = "0.1", path = "./libs/walproposer/" }
|
|
wal_decoder = { version = "0.1", path = "./libs/wal_decoder" }
|
|
|
|
## Common library dependency
|
|
workspace_hack = { version = "0.1", path = "./workspace_hack/" }
|
|
|
|
## Build dependencies
|
|
criterion = "0.5.1"
|
|
rcgen = "0.13"
|
|
rstest = "0.18"
|
|
camino-tempfile = "1.0.2"
|
|
tonic-build = "0.12"
|
|
|
|
[patch.crates-io]
|
|
|
|
# Needed to get `tokio-postgres-rustls` to depend on our fork.
|
|
tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
|
|
|
|
################# Binary contents sections
|
|
|
|
[profile.release]
|
|
# This is useful for profiling and, to some extent, debug.
|
|
# Besides, debug info should not affect the performance.
|
|
debug = true
|
|
|
|
# disable debug symbols for all packages except this one to decrease binaries size
|
|
[profile.release.package."*"]
|
|
debug = false
|
|
|
|
[profile.release-line-debug]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
[profile.release-line-debug-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
lto = true
|
|
|
|
[profile.release-line-debug-size]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
[profile.release-line-debug-zize]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
[profile.release-line-debug-size-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
lto = true
|
|
[profile.release-line-debug-zize-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
lto = true
|
|
|
|
[profile.release-no-debug]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
|
|
[profile.release-no-debug-size]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
[profile.release-no-debug-zize]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
|
|
[profile.release-no-debug-size-lto]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
lto = true
|
|
|
|
[profile.release-no-debug-zize-lto]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
lto = true
|