mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-07 13:32:57 +00:00
07b974480c
## Problem We wish to have high level WAL decoding logic in `wal_decoder::decoder` module. ## Summary of Changes For this we need the `Value` and `NeonWalRecord` types accessible there, so: 1. Move `Value` and `NeonWalRecord` to `pageserver::value` and `pageserver::record` respectively. 2. Get rid of `pageserver::repository` (follow up from (1)) 3. Move PG specific WAL record types to `postgres_ffi::walrecord`. In theory they could live in `wal_decoder`, but it would create a circular dependency between `wal_decoder` and `postgres_ffi`. Long term it makes sense for those types to be PG version specific, so that will work out nicely. 4. Move higher level WAL record types (to be ingested by pageserver) into `wal_decoder::models` Related: https://github.com/neondatabase/neon/issues/9335 Epic: https://github.com/neondatabase/neon/issues/9329
321 lines
11 KiB
TOML
321 lines
11 KiB
TOML
[workspace]
|
|
resolver = "2"
|
|
members = [
|
|
"compute_tools",
|
|
"control_plane",
|
|
"control_plane/storcon_cli",
|
|
"pageserver",
|
|
"pageserver/compaction",
|
|
"pageserver/ctl",
|
|
"pageserver/client",
|
|
"pageserver/pagebench",
|
|
"proxy",
|
|
"safekeeper",
|
|
"storage_broker",
|
|
"storage_controller",
|
|
"storage_controller/client",
|
|
"storage_scrubber",
|
|
"workspace_hack",
|
|
"libs/compute_api",
|
|
"libs/pageserver_api",
|
|
"libs/postgres_ffi",
|
|
"libs/safekeeper_api",
|
|
"libs/desim",
|
|
"libs/utils",
|
|
"libs/consumption_metrics",
|
|
"libs/postgres_backend",
|
|
"libs/pq_proto",
|
|
"libs/tenant_size_model",
|
|
"libs/metrics",
|
|
"libs/postgres_connection",
|
|
"libs/remote_storage",
|
|
"libs/tracing-utils",
|
|
"libs/postgres_ffi/wal_craft",
|
|
"libs/vm_monitor",
|
|
"libs/walproposer",
|
|
"libs/wal_decoder",
|
|
]
|
|
|
|
[workspace.package]
|
|
edition = "2021"
|
|
license = "Apache-2.0"
|
|
|
|
## All dependency versions, used in the project
|
|
[workspace.dependencies]
|
|
ahash = "0.8"
|
|
anyhow = { version = "1.0", features = ["backtrace"] }
|
|
arc-swap = "1.6"
|
|
async-compression = { version = "0.4.0", features = ["tokio", "gzip", "zstd"] }
|
|
atomic-take = "1.1.0"
|
|
azure_core = { version = "0.19", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] }
|
|
azure_identity = { version = "0.19", default-features = false, features = ["enable_reqwest_rustls"] }
|
|
azure_storage = { version = "0.19", default-features = false, features = ["enable_reqwest_rustls"] }
|
|
azure_storage_blobs = { version = "0.19", default-features = false, features = ["enable_reqwest_rustls"] }
|
|
flate2 = "1.0.26"
|
|
async-stream = "0.3"
|
|
async-trait = "0.1"
|
|
aws-config = { version = "1.5", default-features = false, features=["rustls", "sso"] }
|
|
aws-sdk-s3 = "1.52"
|
|
aws-sdk-iam = "1.46.0"
|
|
aws-smithy-async = { version = "1.2.1", default-features = false, features=["rt-tokio"] }
|
|
aws-smithy-types = "1.2"
|
|
aws-credential-types = "1.2.0"
|
|
aws-sigv4 = { version = "1.2", features = ["sign-http"] }
|
|
aws-types = "1.3"
|
|
axum = { version = "0.7.5", features = ["ws"] }
|
|
base64 = "0.13.0"
|
|
bincode = "1.3"
|
|
bindgen = "0.70"
|
|
bit_field = "0.10.2"
|
|
bstr = "1.0"
|
|
byteorder = "1.4"
|
|
bytes = "1.0"
|
|
camino = "1.1.6"
|
|
cfg-if = "1.0.0"
|
|
chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
|
clap = { version = "4.0", features = ["derive"] }
|
|
comfy-table = "7.1"
|
|
const_format = "0.2"
|
|
crc32c = "0.6"
|
|
dashmap = { version = "5.5.0", features = ["raw-api"] }
|
|
either = "1.8"
|
|
enum-map = "2.4.2"
|
|
enumset = "1.0.12"
|
|
fail = "0.5.0"
|
|
fallible-iterator = "0.2"
|
|
framed-websockets = { version = "0.1.0", git = "https://github.com/neondatabase/framed-websockets" }
|
|
futures = "0.3"
|
|
futures-core = "0.3"
|
|
futures-util = "0.3"
|
|
git-version = "0.3"
|
|
hashbrown = "0.14"
|
|
hashlink = "0.9.1"
|
|
hdrhistogram = "7.5.2"
|
|
hex = "0.4"
|
|
hex-literal = "0.4"
|
|
hmac = "0.12.1"
|
|
hostname = "0.4"
|
|
http = {version = "1.1.0", features = ["std"]}
|
|
http-types = { version = "2", default-features = false }
|
|
http-body-util = "0.1.2"
|
|
humantime = "2.1"
|
|
humantime-serde = "1.1.1"
|
|
hyper0 = { package = "hyper", version = "0.14" }
|
|
hyper = "1.4"
|
|
hyper-util = "0.1"
|
|
tokio-tungstenite = "0.21.0"
|
|
indexmap = "2"
|
|
indoc = "2"
|
|
ipnet = "2.9.0"
|
|
itertools = "0.10"
|
|
itoa = "1.0.11"
|
|
jsonwebtoken = "9"
|
|
lasso = "0.7"
|
|
libc = "0.2"
|
|
md5 = "0.7.0"
|
|
measured = { version = "0.0.22", features=["lasso"] }
|
|
measured-process = { version = "0.0.22" }
|
|
memoffset = "0.9"
|
|
nix = { version = "0.27", features = ["dir", "fs", "process", "socket", "signal", "poll"] }
|
|
notify = "6.0.0"
|
|
num_cpus = "1.15"
|
|
num-traits = "0.2.15"
|
|
once_cell = "1.13"
|
|
opentelemetry = "0.24"
|
|
opentelemetry_sdk = "0.24"
|
|
opentelemetry-otlp = { version = "0.17", default-features=false, features = ["http-proto", "trace", "http", "reqwest-client"] }
|
|
opentelemetry-semantic-conventions = "0.16"
|
|
parking_lot = "0.12"
|
|
parquet = { version = "53", default-features = false, features = ["zstd"] }
|
|
parquet_derive = "53"
|
|
pbkdf2 = { version = "0.12.1", features = ["simple", "std"] }
|
|
pin-project-lite = "0.2"
|
|
procfs = "0.16"
|
|
prometheus = {version = "0.13", default-features=false, features = ["process"]} # removes protobuf dependency
|
|
prost = "0.13"
|
|
rand = "0.8"
|
|
redis = { version = "0.25.2", features = ["tokio-rustls-comp", "keep-alive"] }
|
|
regex = "1.10.2"
|
|
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
|
|
reqwest-tracing = { version = "0.5", features = ["opentelemetry_0_24"] }
|
|
reqwest-middleware = "0.3.0"
|
|
reqwest-retry = "0.5"
|
|
routerify = "3"
|
|
rpds = "0.13"
|
|
rustc-hash = "1.1.0"
|
|
rustls = "0.23"
|
|
rustls-pemfile = "2"
|
|
scopeguard = "1.1"
|
|
sysinfo = "0.29.2"
|
|
sd-notify = "0.4.1"
|
|
send-future = "0.1.0"
|
|
sentry = { version = "0.32", default-features = false, features = ["backtrace", "contexts", "panic", "rustls", "reqwest" ] }
|
|
serde = { version = "1.0", features = ["derive"] }
|
|
serde_json = "1"
|
|
serde_path_to_error = "0.1"
|
|
serde_with = "2.0"
|
|
serde_assert = "0.5.0"
|
|
sha2 = "0.10.2"
|
|
signal-hook = "0.3"
|
|
smallvec = "1.11"
|
|
smol_str = { version = "0.2.0", features = ["serde"] }
|
|
socket2 = "0.5"
|
|
strum = "0.26"
|
|
strum_macros = "0.26"
|
|
"subtle" = "2.5.0"
|
|
svg_fmt = "0.4.3"
|
|
sync_wrapper = "0.1.2"
|
|
tar = "0.4"
|
|
test-context = "0.3"
|
|
thiserror = "1.0"
|
|
tikv-jemallocator = "0.5"
|
|
tikv-jemalloc-ctl = "0.5"
|
|
tokio = { version = "1.17", features = ["macros"] }
|
|
tokio-epoll-uring = { git = "https://github.com/neondatabase/tokio-epoll-uring.git" , branch = "main" }
|
|
tokio-io-timeout = "1.2.0"
|
|
tokio-postgres-rustls = "0.12.0"
|
|
tokio-rustls = "0.26"
|
|
tokio-stream = "0.1"
|
|
tokio-tar = "0.3"
|
|
tokio-util = { version = "0.7.10", features = ["io", "rt"] }
|
|
toml = "0.8"
|
|
toml_edit = "0.22"
|
|
tonic = {version = "0.12.3", features = ["tls", "tls-roots"]}
|
|
tower-service = "0.3.2"
|
|
tracing = "0.1"
|
|
tracing-error = "0.2"
|
|
tracing-opentelemetry = "0.25"
|
|
tracing-subscriber = { version = "0.3", default-features = false, features = ["smallvec", "fmt", "tracing-log", "std", "env-filter", "json"] }
|
|
try-lock = "0.2.5"
|
|
twox-hash = { version = "1.6.3", default-features = false }
|
|
typed-json = "0.1"
|
|
url = "2.2"
|
|
urlencoding = "2.1"
|
|
uuid = { version = "1.6.1", features = ["v4", "v7", "serde"] }
|
|
walkdir = "2.3.2"
|
|
rustls-native-certs = "0.8"
|
|
x509-parser = "0.16"
|
|
whoami = "1.5.1"
|
|
|
|
## TODO replace this with tracing
|
|
env_logger = "0.10"
|
|
log = "0.4"
|
|
|
|
## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed
|
|
|
|
# We want to use the 'neon' branch for these, but there's currently one
|
|
# incompatible change on the branch. See:
|
|
#
|
|
# - PR #8076 which contained changes that depended on the new changes in
|
|
# the rust-postgres crate, and
|
|
# - PR #8654 which reverted those changes and made the code in proxy incompatible
|
|
# with the tip of the 'neon' branch again.
|
|
#
|
|
# When those proxy changes are re-applied (see PR #8747), we can switch using
|
|
# the tip of the 'neon' branch again.
|
|
postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev = "20031d7a9ee1addeae6e0968e3899ae6bf01cee2" }
|
|
postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev = "20031d7a9ee1addeae6e0968e3899ae6bf01cee2" }
|
|
postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev = "20031d7a9ee1addeae6e0968e3899ae6bf01cee2" }
|
|
tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev = "20031d7a9ee1addeae6e0968e3899ae6bf01cee2" }
|
|
|
|
## Local libraries
|
|
compute_api = { version = "0.1", path = "./libs/compute_api/" }
|
|
consumption_metrics = { version = "0.1", path = "./libs/consumption_metrics/" }
|
|
metrics = { version = "0.1", path = "./libs/metrics/" }
|
|
pageserver_api = { version = "0.1", path = "./libs/pageserver_api/" }
|
|
pageserver_client = { path = "./pageserver/client" }
|
|
pageserver_compaction = { version = "0.1", path = "./pageserver/compaction/" }
|
|
postgres_backend = { version = "0.1", path = "./libs/postgres_backend/" }
|
|
postgres_connection = { version = "0.1", path = "./libs/postgres_connection/" }
|
|
postgres_ffi = { version = "0.1", path = "./libs/postgres_ffi/" }
|
|
pq_proto = { version = "0.1", path = "./libs/pq_proto/" }
|
|
remote_storage = { version = "0.1", path = "./libs/remote_storage/" }
|
|
safekeeper_api = { version = "0.1", path = "./libs/safekeeper_api" }
|
|
desim = { version = "0.1", path = "./libs/desim" }
|
|
storage_broker = { version = "0.1", path = "./storage_broker/" } # Note: main broker code is inside the binary crate, so linking with the library shouldn't be heavy.
|
|
storage_controller_client = { path = "./storage_controller/client" }
|
|
tenant_size_model = { version = "0.1", path = "./libs/tenant_size_model/" }
|
|
tracing-utils = { version = "0.1", path = "./libs/tracing-utils/" }
|
|
utils = { version = "0.1", path = "./libs/utils/" }
|
|
vm_monitor = { version = "0.1", path = "./libs/vm_monitor/" }
|
|
walproposer = { version = "0.1", path = "./libs/walproposer/" }
|
|
wal_decoder = { version = "0.1", path = "./libs/wal_decoder" }
|
|
|
|
## Common library dependency
|
|
workspace_hack = { version = "0.1", path = "./workspace_hack/" }
|
|
|
|
## Build dependencies
|
|
criterion = "0.5.1"
|
|
rcgen = "0.13"
|
|
rstest = "0.18"
|
|
camino-tempfile = "1.0.2"
|
|
tonic-build = "0.12"
|
|
|
|
[patch.crates-io]
|
|
|
|
# Needed to get `tokio-postgres-rustls` to depend on our fork.
|
|
tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev = "20031d7a9ee1addeae6e0968e3899ae6bf01cee2" }
|
|
|
|
################# Binary contents sections
|
|
|
|
[profile.release]
|
|
# This is useful for profiling and, to some extent, debug.
|
|
# Besides, debug info should not affect the performance.
|
|
debug = true
|
|
|
|
# disable debug symbols for all packages except this one to decrease binaries size
|
|
[profile.release.package."*"]
|
|
debug = false
|
|
|
|
[profile.release-line-debug]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
[profile.release-line-debug-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
lto = true
|
|
|
|
[profile.release-line-debug-size]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
[profile.release-line-debug-zize]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
[profile.release-line-debug-size-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
lto = true
|
|
[profile.release-line-debug-zize-lto]
|
|
inherits = "release"
|
|
debug = 1 # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
lto = true
|
|
|
|
[profile.release-no-debug]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
|
|
[profile.release-no-debug-size]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
[profile.release-no-debug-zize]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
|
|
[profile.release-no-debug-size-lto]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "s"
|
|
lto = true
|
|
|
|
[profile.release-no-debug-zize-lto]
|
|
inherits = "release"
|
|
debug = false # true = 2 = all symbols, 1 = line only
|
|
opt-level = "z"
|
|
lto = true
|