mirror of
https://github.com/neondatabase/neon.git
synced 2025-12-22 21:59:59 +00:00
55aef2993d
See #11992 and #11961 for some examples of usecases. This introduces a JSON serialization lib, designed for more flexibility than serde_json offers. ## Dynamic construction Sometimes you have dynamic values you want to serialize, that are not already in a serde-aware model like a struct or a Vec etc. To achieve this with serde, you need to implement a lot of different traits on a lot of different new-types. Because of this, it's often easier to give-in and pull all the data into a serde-aware model (serde_json::Value or some intermediate struct), but that is often not very efficient. This crate allows full control over the JSON encoding without needing to implement any extra traits. Just call the relevant functions, and it will guarantee a correctly encoded JSON value. ## Async construction Similar to the above, sometimes the values arrive asynchronously. Often collecting those values in memory is more expensive than writing them as JSON, since the overheads of `Vec` and `String` is much higher, however there are exceptions. Serializing to JSON all in one go is also more CPU intensive and can cause lag spikes, whereas serializing values incrementally spreads out the CPU load and reduces lag.
366 lines
13 KiB
TOML
366 lines
13 KiB
TOML
[workspace]
|
|
resolver = "2"
|
|
members = [
|
|
"compute_tools",
|
|
"control_plane",
|
|
"control_plane/storcon_cli",
|
|
"pageserver",
|
|
"pageserver/compaction",
|
|
"pageserver/ctl",
|
|
"pageserver/client",
|
|
"pageserver/pagebench",
|
|
"pageserver/page_api",
|
|
"proxy",
|
|
"safekeeper",
|
|
"safekeeper/client",
|
|
"storage_broker",
|
|
"storage_controller",
|
|
"storage_controller/client",
|
|
"storage_scrubber",
|
|
"workspace_hack",
|
|
"libs/compute_api",
|
|
"libs/http-utils",
|
|
"libs/pageserver_api",
|
|
"libs/postgres_ffi",
|
|
"libs/postgres_ffi_types",
|
|
"libs/postgres_versioninfo",
|
|
"libs/safekeeper_api",
|
|
"libs/desim",
|
|
"libs/neon-shmem",
|
|
"libs/utils",
|
|
"libs/consumption_metrics",
|
|
"libs/postgres_backend",
|
|
"libs/posthog_client_lite",
|
|
"libs/pq_proto",
|
|
"libs/tenant_size_model",
|
|
"libs/metrics",
|
|
"libs/postgres_connection",
|
|
"libs/remote_storage",
|
|
"libs/tracing-utils",
|
|
"libs/postgres_ffi/wal_craft",
|
|
"libs/vm_monitor",
|
|
"libs/walproposer",
|
|
"libs/wal_decoder",
|
|
"libs/postgres_initdb",
|
|
"libs/proxy/json",
|
|
"libs/proxy/postgres-protocol2",
|
|
"libs/proxy/postgres-types2",
|
|
"libs/proxy/tokio-postgres2",
|
|
"endpoint_storage",
|
|
"pgxn/neon/communicator",
|
|
]
|
|
|
|
[workspace.package]
|
|
edition = "2024"
|
|
license = "Apache-2.0"
|
|
|
|
## All dependency versions, used in the project
|
|
[workspace.dependencies]
|
|
ahash = "0.8"
|
|
anyhow = { version = "1.0", features = ["backtrace"] }
|
|
arc-swap = "1.7"
|
|
async-compression = { version = "0.4.0", features = ["tokio", "gzip", "zstd"] }
|
|
atomic-take = "1.1.0"
|
|
flate2 = "1.0.26"
|
|
assert-json-diff = "2"
|
|
async-stream = "0.3"
|
|
async-trait = "0.1"
|
|
aws-config = { version = "1.5", default-features = false, features=["rustls", "sso"] }
|
|
aws-sdk-s3 = "1.52"
|
|
aws-sdk-iam = "1.46.0"
|
|
aws-sdk-kms = "1.47.0"
|
|
aws-smithy-async = { version = "1.2.1", default-features = false, features=["rt-tokio"] }
|
|
aws-smithy-types = "1.2"
|
|
aws-credential-types = "1.2.0"
|
|
aws-sigv4 = { version = "1.2", features = ["sign-http"] }
|
|
aws-types = "1.3"
|
|
axum = { version = "0.8.1", features = ["ws"] }
|
|
axum-extra = { version = "0.10.0", features = ["typed-header", "query"] }
|
|
base64 = "0.22"
|
|
bincode = "1.3"
|
|
bindgen = "0.71"
|
|
bit_field = "0.10.2"
|
|
bstr = "1.0"
|
|
byteorder = "1.4"
|
|
bytes = "1.9"
|
|
camino = "1.1.6"
|
|
cfg-if = "1.0.0"
|
|
cron = "0.15"
|
|
chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
|
clap = { version = "4.0", features = ["derive", "env"] }
|
|
clashmap = { version = "1.0", features = ["raw-api"] }
|
|
comfy-table = "7.1"
|
|
const_format = "0.2"
|
|
crc32c = "0.6"
|
|
diatomic-waker = { version = "0.2.3" }
|
|
either = "1.8"
|
|
enum-map = "2.4.2"
|
|
enumset = "1.0.12"
|
|
fail = "0.5.0"
|
|
fallible-iterator = "0.2"
|
|
framed-websockets = { version = "0.1.0", git = "https://github.com/neondatabase/framed-websockets" }
|
|
futures = "0.3"
|
|
futures-core = "0.3"
|
|
futures-util = "0.3"
|
|
git-version = "0.3"
|
|
governor = "0.8"
|
|
hashbrown = "0.14"
|
|
hashlink = "0.9.1"
|
|
hdrhistogram = "7.5.2"
|
|
hex = "0.4"
|
|
hex-literal = "0.4"
|
|
hmac = "0.12.1"
|
|
hostname = "0.4"
|
|
http = {version = "1.1.0", features = ["std"]}
|
|
http-types = { version = "2", default-features = false }
|
|
http-body-util = "0.1.2"
|
|
humantime = "2.2"
|
|
humantime-serde = "1.1.1"
|
|
hyper0 = { package = "hyper", version = "0.14" }
|
|
hyper = "1.4"
|
|
hyper-util = "0.1"
|
|
tokio-tungstenite = "0.21.0"
|
|
indexmap = { version = "2", features = ["serde"] }
|
|
indoc = "2"
|
|
ipnet = "2.10.0"
|
|
itertools = "0.10"
|
|
itoa = "1.0.11"
|
|
jemalloc_pprof = { version = "0.7", features = ["symbolize", "flamegraph"] }
|
|
jsonwebtoken = "9"
|
|
lasso = "0.7"
|
|
libc = "0.2"
|
|
md5 = "0.7.0"
|
|
measured = { version = "0.0.22", features=["lasso"] }
|
|
measured-process = { version = "0.0.22" }
|
|
memoffset = "0.9"
|
|
nix = { version = "0.30.1", features = ["dir", "fs", "mman", "process", "socket", "signal", "poll"] }
|
|
# Do not update to >= 7.0.0, at least. The update will have a significant impact
|
|
# on compute startup metrics (start_postgres_ms), >= 25% degradation.
|
|
notify = "6.0.0"
|
|
num_cpus = "1.15"
|
|
num-traits = "0.2.19"
|
|
once_cell = "1.13"
|
|
opentelemetry = "0.27"
|
|
opentelemetry_sdk = "0.27"
|
|
opentelemetry-otlp = { version = "0.27", default-features = false, features = ["http-proto", "trace", "http", "reqwest-client"] }
|
|
opentelemetry-semantic-conventions = "0.27"
|
|
parking_lot = "0.12"
|
|
parquet = { version = "53", default-features = false, features = ["zstd"] }
|
|
parquet_derive = "53"
|
|
pbkdf2 = { version = "0.12.1", features = ["simple", "std"] }
|
|
pem = "3.0.3"
|
|
pin-project-lite = "0.2"
|
|
pprof = { version = "0.14", features = ["criterion", "flamegraph", "frame-pointer", "prost-codec"] }
|
|
procfs = "0.16"
|
|
prometheus = {version = "0.13", default-features=false, features = ["process"]} # removes protobuf dependency
|
|
prost = "0.13.5"
|
|
prost-types = "0.13.5"
|
|
rand = "0.8"
|
|
redis = { version = "0.29.2", features = ["tokio-rustls-comp", "keep-alive"] }
|
|
regex = "1.10.2"
|
|
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
|
|
reqwest-tracing = { version = "0.5", features = ["opentelemetry_0_27"] }
|
|
reqwest-middleware = "0.4"
|
|
reqwest-retry = "0.7"
|
|
routerify = "3"
|
|
rpds = "0.13"
|
|
rustc-hash = "1.1.0"
|
|
rustls = { version = "0.23.16", default-features = false }
|
|
rustls-pemfile = "2"
|
|
rustls-pki-types = "1.11"
|
|
scopeguard = "1.1"
|
|
sysinfo = "0.29.2"
|
|
sd-notify = "0.4.1"
|
|
send-future = "0.1.0"
|
|
sentry = { version = "0.37", default-features = false, features = ["backtrace", "contexts", "panic", "rustls", "reqwest" ] }
|
|
serde = { version = "1.0", features = ["derive"] }
|
|
serde_json = "1"
|
|
serde_path_to_error = "0.1"
|
|
serde_with = { version = "3", features = [ "base64" ] }
|
|
serde_assert = "0.5.0"
|
|
serde_repr = "0.1.20"
|
|
sha2 = "0.10.2"
|
|
signal-hook = "0.3"
|
|
smallvec = "1.11"
|
|
smol_str = { version = "0.2.0", features = ["serde"] }
|
|
socket2 = "0.5"
|
|
spki = "0.7.3"
|
|
strum = "0.26"
|
|
strum_macros = "0.26"
|
|
"subtle" = "2.5.0"
|
|
svg_fmt = "0.4.3"
|
|
sync_wrapper = "0.1.2"
|
|
tar = "0.4"
|
|
test-context = "0.3"
|
|
thiserror = "1.0"
|
|
tikv-jemallocator = { version = "0.6", features = ["profiling", "stats", "unprefixed_malloc_on_supported_platforms"] }
|
|
tikv-jemalloc-ctl = { version = "0.6", features = ["stats"] }
|
|
tokio = { version = "1.43.1", features = ["macros"] }
|
|
tokio-epoll-uring = { git = "https://github.com/neondatabase/tokio-epoll-uring.git" , branch = "main" }
|
|
tokio-io-timeout = "1.2.0"
|
|
tokio-postgres-rustls = "0.12.0"
|
|
tokio-rustls = { version = "0.26.0", default-features = false, features = ["tls12", "ring"]}
|
|
tokio-stream = "0.1"
|
|
tokio-tar = "0.3"
|
|
tokio-util = { version = "0.7.10", features = ["io", "io-util", "rt"] }
|
|
toml = "0.8"
|
|
toml_edit = "0.22"
|
|
tonic = { version = "0.13.1", default-features = false, features = ["channel", "codegen", "gzip", "prost", "router", "server", "tls-ring", "tls-native-roots", "zstd"] }
|
|
tonic-reflection = { version = "0.13.1", features = ["server"] }
|
|
tower = { version = "0.5.2", default-features = false }
|
|
tower-http = { version = "0.6.2", features = ["auth", "request-id", "trace"] }
|
|
|
|
# This revision uses opentelemetry 0.27. There's no tag for it.
|
|
tower-otel = { git = "https://github.com/mattiapenati/tower-otel", rev = "56a7321053bcb72443888257b622ba0d43a11fcd" }
|
|
|
|
tower-service = "0.3.3"
|
|
tracing = "0.1"
|
|
tracing-error = "0.2"
|
|
tracing-log = "0.2"
|
|
tracing-opentelemetry = "0.28"
|
|
tracing-serde = "0.2.0"
|
|
tracing-subscriber = { version = "0.3", default-features = false, features = ["smallvec", "fmt", "tracing-log", "std", "env-filter", "json"] }
|
|
try-lock = "0.2.5"
|
|
test-log = { version = "0.2.17", default-features = false, features = ["log"] }
|
|
twox-hash = { version = "1.6.3", default-features = false }
|
|
typed-json = "0.1"
|
|
url = "2.2"
|
|
urlencoding = "2.1"
|
|
uuid = { version = "1.6.1", features = ["v4", "v7", "serde"] }
|
|
walkdir = "2.3.2"
|
|
rustls-native-certs = "0.8"
|
|
whoami = "1.5.1"
|
|
zerocopy = { version = "0.8", features = ["derive", "simd"] }
|
|
json-structural-diff = { version = "0.2.0" }
|
|
x509-cert = { version = "0.2.5" }
|
|
|
|
## TODO replace this with tracing
|
|
env_logger = "0.11"
|
|
log = "0.4"
|
|
|
|
## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed
|
|
postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
|
|
postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
|
|
postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
|
|
tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
|
|
|
|
## Azure SDK crates
|
|
azure_core = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] }
|
|
azure_identity = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] }
|
|
azure_storage = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] }
|
|
azure_storage_blobs = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] }
|
|
|
|
## Local libraries
|
|
compute_api = { version = "0.1", path = "./libs/compute_api/" }
|
|
consumption_metrics = { version = "0.1", path = "./libs/consumption_metrics/" }
|
|
desim = { version = "0.1", path = "./libs/desim" }
|
|
endpoint_storage = { version = "0.0.1", path = "./endpoint_storage/" }
|
|
http-utils = { version = "0.1", path = "./libs/http-utils/" }
|
|
metrics = { version = "0.1", path = "./libs/metrics/" }
|
|
neon-shmem = { version = "0.1", path = "./libs/neon-shmem/" }
|
|
pageserver = { path = "./pageserver" }
|
|
pageserver_api = { version = "0.1", path = "./libs/pageserver_api/" }
|
|
pageserver_client = { path = "./pageserver/client" }
|
|
pageserver_compaction = { version = "0.1", path = "./pageserver/compaction/" }
|
|
pageserver_page_api = { path = "./pageserver/page_api" }
|
|
postgres_backend = { version = "0.1", path = "./libs/postgres_backend/" }
|
|
postgres_connection = { version = "0.1", path = "./libs/postgres_connection/" }
|
|
postgres_ffi = { version = "0.1", path = "./libs/postgres_ffi/" }
|
|
postgres_ffi_types = { version = "0.1", path = "./libs/postgres_ffi_types/" }
|
|
postgres_versioninfo = { version = "0.1", path = "./libs/postgres_versioninfo/" }
|
|
postgres_initdb = { path = "./libs/postgres_initdb" }
|
|
posthog_client_lite = { version = "0.1", path = "./libs/posthog_client_lite" }
|
|
pq_proto = { version = "0.1", path = "./libs/pq_proto/" }
|
|
remote_storage = { version = "0.1", path = "./libs/remote_storage/" }
|
|
safekeeper_api = { version = "0.1", path = "./libs/safekeeper_api" }
|
|
safekeeper_client = { path = "./safekeeper/client" }
|
|
storage_broker = { version = "0.1", path = "./storage_broker/" } # Note: main broker code is inside the binary crate, so linking with the library shouldn't be heavy.
|
|
storage_controller_client = { path = "./storage_controller/client" }
|
|
tenant_size_model = { version = "0.1", path = "./libs/tenant_size_model/" }
|
|
tracing-utils = { version = "0.1", path = "./libs/tracing-utils/" }
|
|
utils = { version = "0.1", path = "./libs/utils/" }
|
|
vm_monitor = { version = "0.1", path = "./libs/vm_monitor/" }
|
|
wal_decoder = { version = "0.1", path = "./libs/wal_decoder" }
|
|
walproposer = { version = "0.1", path = "./libs/walproposer/" }
|
|
|
|
## Common library dependency
|
|
workspace_hack = { version = "0.1", path = "./workspace_hack/" }
|
|
|
|
## Build dependencies
|
|
cbindgen = "0.29.0"
|
|
criterion = "0.5.1"
|
|
rcgen = "0.13"
|
|
rstest = "0.18"
|
|
camino-tempfile = "1.0.2"
|
|
tonic-build = "0.13.1"
|
|
|
|
[patch.crates-io]
|
|
|
|
# Needed to get `tokio-postgres-rustls` to depend on our fork.
|
|
tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
|
|
|
|
################# Binary contents sections
|
|
|
|
[profile.release]
|
|
# This is useful for profiling and, to some extent, debug.
|
|
# Besides, debug info should not affect the performance.
|
|
#
|
|
# NB: we also enable frame pointers for improved profiling, see .cargo/config.toml.
|
|
debug = true
|
|
|
|
# disable debug symbols for all packages except this one to decrease binaries size
|
|
[profile.release.package."*"]
|
|
debug = false
|
|
|
|
[profile.release-line-debug]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
[profile.release-line-debug-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
lto = true
|
|
|
|
[profile.release-line-debug-size]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
[profile.release-line-debug-zize]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
[profile.release-line-debug-size-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
lto = true
|
|
[profile.release-line-debug-zize-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
lto = true
|
|
|
|
[profile.release-no-debug]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
|
|
[profile.release-no-debug-size]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
[profile.release-no-debug-zize]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
|
|
[profile.release-no-debug-size-lto]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
lto = true
|
|
|
|
[profile.release-no-debug-zize-lto]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
lto = true
|