mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-06 13:02:55 +00:00
b992a1a62a
# Summary In - https://github.com/neondatabase/neon/pull/10813 we added slow flush logging but it didn't log the TCP send & recv queue length. This PR adds that data to the log message. I believe the implementation to be safe & correct right now, but it's brittle and thus this PR should be reverted or improved upon once the investigation is over. Refs: - stacked atop https://github.com/neondatabase/neon/pull/10813 - context: https://neondb.slack.com/archives/C08DE6Q9C3B/p1739464533762049?thread_ts=1739462628.361019&cid=C08DE6Q9C3B - improves https://github.com/neondatabase/neon/issues/10668 - part of https://github.com/neondatabase/cloud/issues/23515 # How It Works The trouble is two-fold: 1. getting to the raw socket file descriptor through the many Rust types that wrap it and 2. integrating with the `measure()` function Rust wraps it in types to model file descriptor lifetimes and ownership, and usually one can get access using `as_raw_fd()`. However, we `split()` the stream and the resulting [`tokio::io::WriteHalf`](https://docs.rs/tokio/latest/tokio/io/struct.WriteHalf.html) . Check the PR commit history for my attempts to do it. My solution is to get the socket fd before we wrap it in our protocol types, and to store that fd in the new `PostgresBackend::socket_fd` field. I believe it's safe because the lifetime of `PostgresBackend::socket_fd` value == the lifetime of the `TcpStream` that wrap and store in `PostgresBackend::framed`. Specifically, the only place that close()s the socket is the `impl Drop for TcpStream`. I think the protocol stack calls `TcpStream::shutdown()`, but, that doesn't `close()` the file descriptor underneath. Regarding integration with the `measure()` function, the trouble is that `flush_fut` is currently a generic `Future` type. So, we just pass in the `socket_fd` as a separate argument. A clean implementation would convert the `pgb_writer.flush()` to a named future that provides an accessor for the socket fd while not being polled. I tried (see PR history), but failed to break through the `WriteHalf`. # Testing Tested locally by running ``` ./target/debug/pagebench get-page-latest-lsn --num-clients=1000 --queue-depth=1000 ``` in one terminal, waiting a bit, then ``` pkill -STOP pagebench ``` then wait for slow logs to show up in `pageserver.log`. Pick one of the slow log message's port pairs, e.g., `127.0.0.1:39500`, and then checking sockstat output ``` ss -ntp | grep '127.0.0.1:39500' ``` to ensure that send & recv queue size match those in the log message.
71 lines
1.8 KiB
TOML
71 lines
1.8 KiB
TOML
[package]
|
|
name = "utils"
|
|
version = "0.1.0"
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
|
|
[features]
|
|
default = []
|
|
# Enables test-only APIs, incuding failpoints. In particular, enables the `fail_point!` macro,
|
|
# which adds some runtime cost to run tests on outage conditions
|
|
testing = ["fail/failpoints"]
|
|
|
|
[dependencies]
|
|
arc-swap.workspace = true
|
|
sentry.workspace = true
|
|
async-compression.workspace = true
|
|
anyhow.workspace = true
|
|
backtrace.workspace = true
|
|
bincode.workspace = true
|
|
bytes.workspace = true
|
|
camino.workspace = true
|
|
chrono.workspace = true
|
|
diatomic-waker.workspace = true
|
|
git-version.workspace = true
|
|
hex = { workspace = true, features = ["serde"] }
|
|
humantime.workspace = true
|
|
inferno.workspace = true
|
|
fail.workspace = true
|
|
futures = { workspace = true }
|
|
jsonwebtoken.workspace = true
|
|
nix = {workspace = true, features = [ "ioctl" ] }
|
|
once_cell.workspace = true
|
|
pin-project-lite.workspace = true
|
|
regex.workspace = true
|
|
serde.workspace = true
|
|
serde_with.workspace = true
|
|
serde_json.workspace = true
|
|
signal-hook.workspace = true
|
|
thiserror.workspace = true
|
|
tokio.workspace = true
|
|
tokio-tar.workspace = true
|
|
tokio-util.workspace = true
|
|
toml_edit = { workspace = true, features = ["serde"] }
|
|
tracing.workspace = true
|
|
tracing-error.workspace = true
|
|
tracing-subscriber = { workspace = true, features = ["json", "registry"] }
|
|
rand.workspace = true
|
|
scopeguard.workspace = true
|
|
strum.workspace = true
|
|
strum_macros.workspace = true
|
|
walkdir.workspace = true
|
|
|
|
pq_proto.workspace = true
|
|
postgres_connection.workspace = true
|
|
metrics.workspace = true
|
|
|
|
const_format.workspace = true
|
|
|
|
[dev-dependencies]
|
|
byteorder.workspace = true
|
|
bytes.workspace = true
|
|
criterion.workspace = true
|
|
hex-literal.workspace = true
|
|
camino-tempfile.workspace = true
|
|
serde_assert.workspace = true
|
|
tokio = { workspace = true, features = ["test-util"] }
|
|
|
|
[[bench]]
|
|
name = "benchmarks"
|
|
harness = false
|