mirror of
https://github.com/neondatabase/neon.git
synced 2026-05-19 14:10:37 +00:00
8a646cb750
## Summary of changes ### RequestMonitoring We want to add an event stream with information on each request for easier analysis than what we can do with diagnostic logs alone (https://github.com/neondatabase/cloud/issues/8807). This RequestMonitoring will keep a record of the final state of a request. On drop it will be pushed into a queue to be uploaded. Because this context is a bag of data, I don't want this information to impact logic of request handling. I personally think that weakly typed data (such as all these options) makes for spaghetti code. I will however allow for this data to impact rate-limiting and blocking of requests, as this does not _really_ change how a request is handled. ### Parquet Each `RequestMonitoring` is flushed into a channel where it is converted into `RequestData`, which is accumulated into parquet files. Each file will have a certain number of rows per row group, and several row groups will eventually fill up the file, which we then upload to S3. We will also upload smaller files if they take too long to construct.
286 lines
8.9 KiB
TOML
286 lines
8.9 KiB
TOML
[workspace]
|
|
resolver = "2"
|
|
members = [
|
|
"compute_tools",
|
|
"control_plane",
|
|
"pageserver",
|
|
"pageserver/ctl",
|
|
"pageserver/client",
|
|
"pageserver/pagebench",
|
|
"proxy",
|
|
"safekeeper",
|
|
"storage_broker",
|
|
"s3_scrubber",
|
|
"workspace_hack",
|
|
"trace",
|
|
"libs/compute_api",
|
|
"libs/pageserver_api",
|
|
"libs/postgres_ffi",
|
|
"libs/safekeeper_api",
|
|
"libs/utils",
|
|
"libs/consumption_metrics",
|
|
"libs/postgres_backend",
|
|
"libs/pq_proto",
|
|
"libs/tenant_size_model",
|
|
"libs/metrics",
|
|
"libs/postgres_connection",
|
|
"libs/remote_storage",
|
|
"libs/tracing-utils",
|
|
"libs/postgres_ffi/wal_craft",
|
|
"libs/vm_monitor",
|
|
"libs/walproposer",
|
|
]
|
|
|
|
[workspace.package]
|
|
edition = "2021"
|
|
license = "Apache-2.0"
|
|
|
|
## All dependency versions, used in the project
|
|
[workspace.dependencies]
|
|
anyhow = { version = "1.0", features = ["backtrace"] }
|
|
arc-swap = "1.6"
|
|
async-compression = { version = "0.4.0", features = ["tokio", "gzip", "zstd"] }
|
|
azure_core = "0.18"
|
|
azure_identity = "0.18"
|
|
azure_storage = "0.18"
|
|
azure_storage_blobs = "0.18"
|
|
flate2 = "1.0.26"
|
|
async-stream = "0.3"
|
|
async-trait = "0.1"
|
|
aws-config = { version = "1.0", default-features = false, features=["rustls"] }
|
|
aws-sdk-s3 = "1.0"
|
|
aws-smithy-async = { version = "1.0", default-features = false, features=["rt-tokio"] }
|
|
aws-smithy-types = "1.0"
|
|
aws-credential-types = "1.0"
|
|
axum = { version = "0.6.20", features = ["ws"] }
|
|
base64 = "0.13.0"
|
|
bincode = "1.3"
|
|
bindgen = "0.65"
|
|
bstr = "1.0"
|
|
byteorder = "1.4"
|
|
bytes = "1.0"
|
|
camino = "1.1.6"
|
|
cfg-if = "1.0.0"
|
|
chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
|
clap = { version = "4.0", features = ["derive"] }
|
|
close_fds = "0.3.2"
|
|
comfy-table = "6.1"
|
|
const_format = "0.2"
|
|
crc32c = "0.6"
|
|
crossbeam-utils = "0.8.5"
|
|
dashmap = { version = "5.5.0", features = ["raw-api"] }
|
|
either = "1.8"
|
|
enum-map = "2.4.2"
|
|
enumset = "1.0.12"
|
|
fail = "0.5.0"
|
|
fs2 = "0.4.3"
|
|
futures = "0.3"
|
|
futures-core = "0.3"
|
|
futures-util = "0.3"
|
|
git-version = "0.3"
|
|
hashbrown = "0.13"
|
|
hashlink = "0.8.1"
|
|
hdrhistogram = "7.5.2"
|
|
hex = "0.4"
|
|
hex-literal = "0.4"
|
|
hmac = "0.12.1"
|
|
hostname = "0.3.1"
|
|
http-types = { version = "2", default-features = false }
|
|
humantime = "2.1"
|
|
humantime-serde = "1.1.1"
|
|
hyper = "0.14"
|
|
hyper-tungstenite = "0.11"
|
|
inotify = "0.10.2"
|
|
ipnet = "2.9.0"
|
|
itertools = "0.10"
|
|
jsonwebtoken = "9"
|
|
libc = "0.2"
|
|
md5 = "0.7.0"
|
|
memoffset = "0.8"
|
|
native-tls = "0.2"
|
|
nix = "0.26"
|
|
notify = "5.0.0"
|
|
num_cpus = "1.15"
|
|
num-traits = "0.2.15"
|
|
once_cell = "1.13"
|
|
opentelemetry = "0.19.0"
|
|
opentelemetry-otlp = { version = "0.12.0", default_features=false, features = ["http-proto", "trace", "http", "reqwest-client"] }
|
|
opentelemetry-semantic-conventions = "0.11.0"
|
|
parking_lot = "0.12"
|
|
parquet = { version = "49.0.0", default-features = false, features = ["zstd"] }
|
|
parquet_derive = "49.0.0"
|
|
pbkdf2 = { version = "0.12.1", features = ["simple", "std"] }
|
|
pin-project-lite = "0.2"
|
|
prometheus = {version = "0.13", default_features=false, features = ["process"]} # removes protobuf dependency
|
|
prost = "0.11"
|
|
rand = "0.8"
|
|
regex = "1.10.2"
|
|
reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] }
|
|
reqwest-tracing = { version = "0.4.0", features = ["opentelemetry_0_19"] }
|
|
reqwest-middleware = "0.2.0"
|
|
reqwest-retry = "0.2.2"
|
|
routerify = "3"
|
|
rpds = "0.13"
|
|
rustc-hash = "1.1.0"
|
|
rustls = "0.21"
|
|
rustls-pemfile = "1"
|
|
rustls-split = "0.3"
|
|
scopeguard = "1.1"
|
|
sysinfo = "0.29.2"
|
|
sd-notify = "0.4.1"
|
|
sentry = { version = "0.31", default-features = false, features = ["backtrace", "contexts", "panic", "rustls", "reqwest" ] }
|
|
serde = { version = "1.0", features = ["derive"] }
|
|
serde_json = "1"
|
|
serde_path_to_error = "0.1"
|
|
serde_with = "2.0"
|
|
serde_assert = "0.5.0"
|
|
sha2 = "0.10.2"
|
|
signal-hook = "0.3"
|
|
smallvec = "1.11"
|
|
smol_str = { version = "0.2.0", features = ["serde"] }
|
|
socket2 = "0.5"
|
|
strum = "0.24"
|
|
strum_macros = "0.24"
|
|
svg_fmt = "0.4.1"
|
|
sync_wrapper = "0.1.2"
|
|
tar = "0.4"
|
|
task-local-extensions = "0.1.4"
|
|
test-context = "0.1"
|
|
thiserror = "1.0"
|
|
tls-listener = { version = "0.7", features = ["rustls", "hyper-h1"] }
|
|
tokio = { version = "1.17", features = ["macros"] }
|
|
tokio-io-timeout = "1.2.0"
|
|
tokio-postgres-rustls = "0.10.0"
|
|
tokio-rustls = "0.24"
|
|
tokio-stream = "0.1"
|
|
tokio-tar = "0.3"
|
|
tokio-util = { version = "0.7.10", features = ["io", "rt"] }
|
|
toml = "0.7"
|
|
toml_edit = "0.19"
|
|
tonic = {version = "0.9", features = ["tls", "tls-roots"]}
|
|
tracing = "0.1"
|
|
tracing-error = "0.2.0"
|
|
tracing-opentelemetry = "0.19.0"
|
|
tracing-subscriber = { version = "0.3", default_features = false, features = ["smallvec", "fmt", "tracing-log", "std", "env-filter", "json"] }
|
|
url = "2.2"
|
|
uuid = { version = "1.6.1", features = ["v4", "v7", "serde"] }
|
|
walkdir = "2.3.2"
|
|
webpki-roots = "0.25"
|
|
x509-parser = "0.15"
|
|
|
|
## TODO replace this with tracing
|
|
env_logger = "0.10"
|
|
log = "0.4"
|
|
|
|
## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed
|
|
postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch="neon" }
|
|
postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", branch="neon" }
|
|
postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", branch="neon" }
|
|
postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", branch="neon" }
|
|
tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch="neon" }
|
|
|
|
## Other git libraries
|
|
heapless = { default-features=false, features=[], git = "https://github.com/japaric/heapless.git", rev = "644653bf3b831c6bb4963be2de24804acf5e5001" } # upstream release pending
|
|
|
|
## Local libraries
|
|
compute_api = { version = "0.1", path = "./libs/compute_api/" }
|
|
consumption_metrics = { version = "0.1", path = "./libs/consumption_metrics/" }
|
|
metrics = { version = "0.1", path = "./libs/metrics/" }
|
|
pageserver_api = { version = "0.1", path = "./libs/pageserver_api/" }
|
|
pageserver_client = { path = "./pageserver/client" }
|
|
postgres_backend = { version = "0.1", path = "./libs/postgres_backend/" }
|
|
postgres_connection = { version = "0.1", path = "./libs/postgres_connection/" }
|
|
postgres_ffi = { version = "0.1", path = "./libs/postgres_ffi/" }
|
|
pq_proto = { version = "0.1", path = "./libs/pq_proto/" }
|
|
remote_storage = { version = "0.1", path = "./libs/remote_storage/" }
|
|
safekeeper_api = { version = "0.1", path = "./libs/safekeeper_api" }
|
|
storage_broker = { version = "0.1", path = "./storage_broker/" } # Note: main broker code is inside the binary crate, so linking with the library shouldn't be heavy.
|
|
tenant_size_model = { version = "0.1", path = "./libs/tenant_size_model/" }
|
|
tracing-utils = { version = "0.1", path = "./libs/tracing-utils/" }
|
|
utils = { version = "0.1", path = "./libs/utils/" }
|
|
vm_monitor = { version = "0.1", path = "./libs/vm_monitor/" }
|
|
walproposer = { version = "0.1", path = "./libs/walproposer/" }
|
|
|
|
## Common library dependency
|
|
workspace_hack = { version = "0.1", path = "./workspace_hack/" }
|
|
|
|
## Build dependencies
|
|
criterion = "0.5.1"
|
|
rcgen = "0.11"
|
|
rstest = "0.18"
|
|
camino-tempfile = "1.0.2"
|
|
tonic-build = "0.9"
|
|
|
|
[patch.crates-io]
|
|
|
|
# This is only needed for proxy's tests.
|
|
# TODO: we should probably fork `tokio-postgres-rustls` instead.
|
|
tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch="neon" }
|
|
|
|
# bug fixes for UUID
|
|
parquet = { git = "https://github.com/neondatabase/arrow-rs", branch = "neon-fix-bugs" }
|
|
parquet_derive = { git = "https://github.com/neondatabase/arrow-rs", branch = "neon-fix-bugs" }
|
|
|
|
################# Binary contents sections
|
|
|
|
[profile.release]
|
|
# This is useful for profiling and, to some extent, debug.
|
|
# Besides, debug info should not affect the performance.
|
|
debug = true
|
|
|
|
# disable debug symbols for all packages except this one to decrease binaries size
|
|
[profile.release.package."*"]
|
|
debug = false
|
|
|
|
[profile.release-line-debug]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
[profile.release-line-debug-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
lto = true
|
|
|
|
[profile.release-line-debug-size]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
[profile.release-line-debug-zize]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
[profile.release-line-debug-size-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
lto = true
|
|
[profile.release-line-debug-zize-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
lto = true
|
|
|
|
[profile.release-no-debug]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
|
|
[profile.release-no-debug-size]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
[profile.release-no-debug-zize]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
|
|
[profile.release-no-debug-size-lto]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
lto = true
|
|
|
|
[profile.release-no-debug-zize-lto]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
lto = true
|