mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-07 21:42:56 +00:00
d8b0c0834e
Exposes an endpoint "/profile/cpu" for profiling the postgres processes (currently spawned and the new ones) using "perf". Adds the corresponding python test to test the added endpoint and confirm the output expected is the profiling data in the expected format. Add "perf" binary to the sudo list. Fix python poetry ruff Address the clippy lints Document the code Format python code Address code review Prettify Embed profile_pb2.py and small code/test fixes. Make the code slightly better. 1. Makes optional the sampling_frequency parameter for profiling. 2. Avoids using unsafe code when killing a child. Better code, better tests More tests Separate start and stop of profiling Correctly check for the exceptions Address clippy lint Final fixes. 1. Allows the perf to be found in $PATH instead of having the path hardcoded. 2. Changes the path to perf in the sudoers file so that the compute can run it properly. 3. Changes the way perf is invoked, now it is with sudo and the path from $PATH. 4. Removes the authentication requirement from the /profile/cpu/ endpoint. hakari thing Python fixes Fix python formatting More python fixes Update poetry lock Fix ruff Address the review comments Fix the tests Try fixing the flaky test for pg17? Try fixing the flaky test for pg17? PYTHON Fix the tests Remove the PROGRESS parameter Remove unused Increase the timeout due to concurrency Increase the timeout to 60 Increase the profiling window timeout Try this Lets see the error Just log all the errors Add perf into the build environment uijdfghjdf Update tempfile to 3.20 Snapshot Use bbc-profile Update tempfile to 3.20 Provide bpfcc-tools in debian Properly respond with status Python check Fix build-tools dockerfile Add path probation for the bcc profile Try err printing Refactor Add bpfcc-tools to the final image Add error context sudo not found? Print more errors for verbosity Remove procfs and use libproc Update hakari Debug sudo in CI Rebase and adjust hakari remove leftover Add archiving support Correct the paths to the perf binary Try hardcoded sudo path Add sudo into build-tools dockerfile Minor cleanup Print out the sudoers file from github Stop the tests earlier Add the sudoers entry for nonroot, install kmod for modprobe for bcc-profile Try hacking the kernel headers for bcc-profile Redeclare the kernel version argument Try using the kernel of the runner Try another way Check bpfcc-tools
370 lines
13 KiB
TOML
370 lines
13 KiB
TOML
[workspace]
|
|
resolver = "2"
|
|
members = [
|
|
"compute_tools",
|
|
"control_plane",
|
|
"control_plane/storcon_cli",
|
|
"pageserver",
|
|
"pageserver/compaction",
|
|
"pageserver/ctl",
|
|
"pageserver/client",
|
|
"pageserver/client_grpc",
|
|
"pageserver/pagebench",
|
|
"pageserver/page_api",
|
|
"proxy",
|
|
"safekeeper",
|
|
"safekeeper/client",
|
|
"storage_broker",
|
|
"storage_controller",
|
|
"storage_controller/client",
|
|
"storage_scrubber",
|
|
"workspace_hack",
|
|
"libs/compute_api",
|
|
"libs/http-utils",
|
|
"libs/pageserver_api",
|
|
"libs/postgres_ffi",
|
|
"libs/postgres_ffi_types",
|
|
"libs/postgres_versioninfo",
|
|
"libs/safekeeper_api",
|
|
"libs/desim",
|
|
"libs/neon-shmem",
|
|
"libs/utils",
|
|
"libs/consumption_metrics",
|
|
"libs/postgres_backend",
|
|
"libs/posthog_client_lite",
|
|
"libs/pq_proto",
|
|
"libs/tenant_size_model",
|
|
"libs/metrics",
|
|
"libs/postgres_connection",
|
|
"libs/remote_storage",
|
|
"libs/tracing-utils",
|
|
"libs/postgres_ffi/wal_craft",
|
|
"libs/vm_monitor",
|
|
"libs/walproposer",
|
|
"libs/wal_decoder",
|
|
"libs/postgres_initdb",
|
|
"libs/proxy/json",
|
|
"libs/proxy/postgres-protocol2",
|
|
"libs/proxy/postgres-types2",
|
|
"libs/proxy/tokio-postgres2",
|
|
"endpoint_storage",
|
|
"pgxn/neon/communicator",
|
|
]
|
|
|
|
[workspace.package]
|
|
edition = "2024"
|
|
license = "Apache-2.0"
|
|
|
|
## All dependency versions, used in the project
|
|
[workspace.dependencies]
|
|
ahash = "0.8"
|
|
anyhow = { version = "1.0", features = ["backtrace"] }
|
|
arc-swap = "1.7"
|
|
async-compression = { version = "0.4.0", features = ["tokio", "gzip", "zstd"] }
|
|
atomic-take = "1.1.0"
|
|
flate2 = "1.0.26"
|
|
assert-json-diff = "2"
|
|
async-stream = "0.3"
|
|
async-trait = "0.1"
|
|
aws-config = { version = "1.5", default-features = false, features=["rustls", "sso"] }
|
|
aws-sdk-s3 = "1.52"
|
|
aws-sdk-iam = "1.46.0"
|
|
aws-sdk-kms = "1.47.0"
|
|
aws-smithy-async = { version = "1.2.1", default-features = false, features=["rt-tokio"] }
|
|
aws-smithy-types = "1.2"
|
|
aws-credential-types = "1.2.0"
|
|
aws-sigv4 = { version = "1.2", features = ["sign-http"] }
|
|
aws-types = "1.3"
|
|
axum = { version = "0.8.1", features = ["ws"] }
|
|
axum-extra = { version = "0.10.0", features = ["typed-header", "query"] }
|
|
base64 = "0.22"
|
|
bincode = "1.3"
|
|
bindgen = "0.71"
|
|
bit_field = "0.10.2"
|
|
bstr = "1.0"
|
|
byteorder = "1.4"
|
|
bytes = "1.9"
|
|
camino = "1.1.6"
|
|
cfg-if = "1.0.0"
|
|
cron = "0.15"
|
|
chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
|
clap = { version = "4.0", features = ["derive", "env"] }
|
|
clashmap = { version = "1.0", features = ["raw-api"] }
|
|
comfy-table = "7.1"
|
|
const_format = "0.2"
|
|
crc32c = "0.6"
|
|
diatomic-waker = { version = "0.2.3" }
|
|
either = "1.8"
|
|
enum-map = "2.4.2"
|
|
enumset = "1.0.12"
|
|
fail = "0.5.0"
|
|
fallible-iterator = "0.2"
|
|
framed-websockets = { version = "0.1.0", git = "https://github.com/neondatabase/framed-websockets" }
|
|
futures = "0.3"
|
|
futures-core = "0.3"
|
|
futures-util = "0.3"
|
|
git-version = "0.3"
|
|
governor = "0.8"
|
|
hashbrown = "0.14"
|
|
hashlink = "0.9.1"
|
|
hdrhistogram = "7.5.2"
|
|
hex = "0.4"
|
|
hex-literal = "0.4"
|
|
hmac = "0.12.1"
|
|
hostname = "0.4"
|
|
http = {version = "1.1.0", features = ["std"]}
|
|
http-types = { version = "2", default-features = false }
|
|
http-body-util = "0.1.2"
|
|
humantime = "2.2"
|
|
humantime-serde = "1.1.1"
|
|
hyper0 = { package = "hyper", version = "0.14" }
|
|
hyper = "1.4"
|
|
hyper-util = "0.1"
|
|
tokio-tungstenite = "0.21.0"
|
|
indexmap = { version = "2", features = ["serde"] }
|
|
indoc = "2"
|
|
ipnet = "2.10.0"
|
|
itertools = "0.10"
|
|
itoa = "1.0.11"
|
|
jemalloc_pprof = { version = "0.7", features = ["symbolize", "flamegraph"] }
|
|
jsonwebtoken = "9"
|
|
lasso = "0.7"
|
|
libc = "0.2"
|
|
libproc = "0.14"
|
|
md5 = "0.7.0"
|
|
measured = { version = "0.0.22", features=["lasso"] }
|
|
measured-process = { version = "0.0.22" }
|
|
memoffset = "0.9"
|
|
nix = { version = "0.30.1", features = ["dir", "fs", "mman", "process", "socket", "signal", "poll"] }
|
|
# Do not update to >= 7.0.0, at least. The update will have a significant impact
|
|
# on compute startup metrics (start_postgres_ms), >= 25% degradation.
|
|
notify = "6.0.0"
|
|
num_cpus = "1.15"
|
|
num-traits = "0.2.19"
|
|
once_cell = "1.13"
|
|
opentelemetry = "0.27"
|
|
opentelemetry_sdk = "0.27"
|
|
opentelemetry-otlp = { version = "0.27", default-features = false, features = ["http-proto", "trace", "http", "reqwest-client"] }
|
|
opentelemetry-semantic-conventions = "0.27"
|
|
parking_lot = "0.12"
|
|
parquet = { version = "53", default-features = false, features = ["zstd"] }
|
|
parquet_derive = "53"
|
|
pbkdf2 = { version = "0.12.1", features = ["simple", "std"] }
|
|
pem = "3.0.3"
|
|
pin-project-lite = "0.2"
|
|
pprof = { version = "0.14", features = ["criterion", "flamegraph", "frame-pointer", "prost-codec"] }
|
|
procfs = "0.16"
|
|
prometheus = {version = "0.13", default-features=false, features = ["process"]} # removes protobuf dependency
|
|
prost = "0.13.5"
|
|
prost-types = "0.13.5"
|
|
rand = "0.8"
|
|
redis = { version = "0.29.2", features = ["tokio-rustls-comp", "keep-alive"] }
|
|
regex = "1.10.2"
|
|
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
|
|
reqwest-tracing = { version = "0.5", features = ["opentelemetry_0_27"] }
|
|
reqwest-middleware = "0.4"
|
|
reqwest-retry = "0.7"
|
|
routerify = "3"
|
|
rpds = "0.13"
|
|
rustc-hash = "1.1.0"
|
|
rustls = { version = "0.23.16", default-features = false }
|
|
rustls-pemfile = "2"
|
|
rustls-pki-types = "1.11"
|
|
scopeguard = "1.1"
|
|
sysinfo = "0.29.2"
|
|
sd-notify = "0.4.1"
|
|
send-future = "0.1.0"
|
|
sentry = { version = "0.37", default-features = false, features = ["backtrace", "contexts", "panic", "rustls", "reqwest" ] }
|
|
serde = { version = "1.0", features = ["derive"] }
|
|
serde_json = "1"
|
|
serde_path_to_error = "0.1"
|
|
serde_with = { version = "3", features = [ "base64" ] }
|
|
serde_assert = "0.5.0"
|
|
serde_repr = "0.1.20"
|
|
sha2 = "0.10.2"
|
|
signal-hook = "0.3"
|
|
smallvec = "1.11"
|
|
smol_str = { version = "0.2.0", features = ["serde"] }
|
|
socket2 = "0.5"
|
|
spki = "0.7.3"
|
|
strum = "0.26"
|
|
strum_macros = "0.26"
|
|
"subtle" = "2.5.0"
|
|
svg_fmt = "0.4.3"
|
|
sync_wrapper = "0.1.2"
|
|
tar = "0.4"
|
|
test-context = "0.3"
|
|
thiserror = "1.0"
|
|
tikv-jemallocator = { version = "0.6", features = ["profiling", "stats", "unprefixed_malloc_on_supported_platforms"] }
|
|
tikv-jemalloc-ctl = { version = "0.6", features = ["stats"] }
|
|
tokio = { version = "1.43.1", features = ["macros"] }
|
|
tokio-epoll-uring = { git = "https://github.com/neondatabase/tokio-epoll-uring.git" , branch = "main" }
|
|
tokio-io-timeout = "1.2.0"
|
|
tokio-postgres-rustls = "0.12.0"
|
|
tokio-rustls = { version = "0.26.0", default-features = false, features = ["tls12", "ring"]}
|
|
tokio-stream = "0.1"
|
|
tokio-tar = "0.3"
|
|
tokio-util = { version = "0.7.10", features = ["io", "io-util", "rt"] }
|
|
toml = "0.8"
|
|
toml_edit = "0.22"
|
|
tonic = { version = "0.13.1", default-features = false, features = ["channel", "codegen", "gzip", "prost", "router", "server", "tls-ring", "tls-native-roots", "zstd"] }
|
|
tonic-reflection = { version = "0.13.1", features = ["server"] }
|
|
tower = { version = "0.5.2", default-features = false }
|
|
tower-http = { version = "0.6.2", features = ["auth", "request-id", "trace"] }
|
|
|
|
# This revision uses opentelemetry 0.27. There's no tag for it.
|
|
tower-otel = { git = "https://github.com/mattiapenati/tower-otel", rev = "56a7321053bcb72443888257b622ba0d43a11fcd" }
|
|
|
|
tower-service = "0.3.3"
|
|
tracing = "0.1"
|
|
tracing-error = "0.2"
|
|
tracing-log = "0.2"
|
|
tracing-opentelemetry = "0.28"
|
|
tracing-serde = "0.2.0"
|
|
tracing-subscriber = { version = "0.3", default-features = false, features = ["smallvec", "fmt", "tracing-log", "std", "env-filter", "json"] }
|
|
try-lock = "0.2.5"
|
|
test-log = { version = "0.2.17", default-features = false, features = ["log"] }
|
|
twox-hash = { version = "1.6.3", default-features = false }
|
|
typed-json = "0.1"
|
|
url = "2.2"
|
|
urlencoding = "2.1"
|
|
uuid = { version = "1.6.1", features = ["v4", "v7", "serde"] }
|
|
walkdir = "2.3.2"
|
|
rustls-native-certs = "0.8"
|
|
whoami = "1.5.1"
|
|
zerocopy = { version = "0.8", features = ["derive", "simd"] }
|
|
json-structural-diff = { version = "0.2.0" }
|
|
x509-cert = { version = "0.2.5" }
|
|
|
|
## TODO replace this with tracing
|
|
env_logger = "0.11"
|
|
log = "0.4"
|
|
|
|
## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed
|
|
postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
|
|
postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
|
|
postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
|
|
tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
|
|
|
|
## Azure SDK crates
|
|
azure_core = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] }
|
|
azure_identity = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] }
|
|
azure_storage = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] }
|
|
azure_storage_blobs = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] }
|
|
|
|
## Local libraries
|
|
compute_api = { version = "0.1", path = "./libs/compute_api/" }
|
|
consumption_metrics = { version = "0.1", path = "./libs/consumption_metrics/" }
|
|
desim = { version = "0.1", path = "./libs/desim" }
|
|
endpoint_storage = { version = "0.0.1", path = "./endpoint_storage/" }
|
|
http-utils = { version = "0.1", path = "./libs/http-utils/" }
|
|
metrics = { version = "0.1", path = "./libs/metrics/" }
|
|
neon-shmem = { version = "0.1", path = "./libs/neon-shmem/" }
|
|
pageserver = { path = "./pageserver" }
|
|
pageserver_api = { version = "0.1", path = "./libs/pageserver_api/" }
|
|
pageserver_client = { path = "./pageserver/client" }
|
|
pageserver_client_grpc = { path = "./pageserver/client_grpc" }
|
|
pageserver_compaction = { version = "0.1", path = "./pageserver/compaction/" }
|
|
pageserver_page_api = { path = "./pageserver/page_api" }
|
|
postgres_backend = { version = "0.1", path = "./libs/postgres_backend/" }
|
|
postgres_connection = { version = "0.1", path = "./libs/postgres_connection/" }
|
|
postgres_ffi = { version = "0.1", path = "./libs/postgres_ffi/" }
|
|
postgres_ffi_types = { version = "0.1", path = "./libs/postgres_ffi_types/" }
|
|
postgres_versioninfo = { version = "0.1", path = "./libs/postgres_versioninfo/" }
|
|
postgres_initdb = { path = "./libs/postgres_initdb" }
|
|
posthog_client_lite = { version = "0.1", path = "./libs/posthog_client_lite" }
|
|
pq_proto = { version = "0.1", path = "./libs/pq_proto/" }
|
|
remote_storage = { version = "0.1", path = "./libs/remote_storage/" }
|
|
safekeeper_api = { version = "0.1", path = "./libs/safekeeper_api" }
|
|
safekeeper_client = { path = "./safekeeper/client" }
|
|
storage_broker = { version = "0.1", path = "./storage_broker/" } # Note: main broker code is inside the binary crate, so linking with the library shouldn't be heavy.
|
|
storage_controller_client = { path = "./storage_controller/client" }
|
|
tempfile = "3"
|
|
tenant_size_model = { version = "0.1", path = "./libs/tenant_size_model/" }
|
|
tracing-utils = { version = "0.1", path = "./libs/tracing-utils/" }
|
|
utils = { version = "0.1", path = "./libs/utils/" }
|
|
vm_monitor = { version = "0.1", path = "./libs/vm_monitor/" }
|
|
wal_decoder = { version = "0.1", path = "./libs/wal_decoder" }
|
|
walproposer = { version = "0.1", path = "./libs/walproposer/" }
|
|
|
|
## Common library dependency
|
|
workspace_hack = { version = "0.1", path = "./workspace_hack/" }
|
|
|
|
## Build dependencies
|
|
cbindgen = "0.29.0"
|
|
criterion = "0.5.1"
|
|
rcgen = "0.13"
|
|
rstest = "0.18"
|
|
camino-tempfile = "1.0.2"
|
|
tonic-build = "0.13.1"
|
|
|
|
[patch.crates-io]
|
|
|
|
# Needed to get `tokio-postgres-rustls` to depend on our fork.
|
|
tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
|
|
|
|
################# Binary contents sections
|
|
|
|
[profile.release]
|
|
# This is useful for profiling and, to some extent, debug.
|
|
# Besides, debug info should not affect the performance.
|
|
#
|
|
# NB: we also enable frame pointers for improved profiling, see .cargo/config.toml.
|
|
debug = true
|
|
|
|
# disable debug symbols for all packages except this one to decrease binaries size
|
|
[profile.release.package."*"]
|
|
debug = false
|
|
|
|
[profile.release-line-debug]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
[profile.release-line-debug-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
lto = true
|
|
|
|
[profile.release-line-debug-size]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
[profile.release-line-debug-zize]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
[profile.release-line-debug-size-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
lto = true
|
|
[profile.release-line-debug-zize-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
lto = true
|
|
|
|
[profile.release-no-debug]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
|
|
[profile.release-no-debug-size]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
[profile.release-no-debug-zize]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
|
|
[profile.release-no-debug-size-lto]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
lto = true
|
|
|
|
[profile.release-no-debug-zize-lto]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
lto = true
|