diff --git a/.cargo/config.toml b/.cargo/config.toml index c71d491303..44b154b7a2 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -3,15 +3,18 @@ # by the RUSTDOCFLAGS env var in CI. rustdocflags = ["-Arustdoc::private_intra_doc_links"] -# Enable frame pointers. This may have a minor performance overhead, but makes it easier and more -# efficient to obtain stack traces (and thus CPU/heap profiles). It may also avoid seg faults that -# we've seen with libunwind-based profiling. See also: -# -# * -# * -# # NB: the RUSTFLAGS envvar will replace this. Make sure to update e.g. Dockerfile as well. -rustflags = ["-Cforce-frame-pointers=yes"] +rustflags = [ + # Enable frame pointers. This may have a minor performance overhead, but makes it easier and more + # efficient to obtain stack traces (and thus CPU/heap profiles). It may also avoid seg faults that + # we've seen with libunwind-based profiling. See also: + # + # * + # * + "-Cforce-frame-pointers=yes", + # Enable tokio_unstable to enable tokio-console support + "--cfg=tokio_unstable" +] [alias] build_testing = ["build", "--features", "testing"] diff --git a/Cargo.lock b/Cargo.lock index de1b1218ca..530f4b7116 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -717,13 +717,40 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core 0.4.5", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "itoa", + "matchit 0.7.3", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper 1.0.1", + "tower 0.5.2", + "tower-layer", + "tower-service", +] + [[package]] name = "axum" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d6fd624c75e18b3b4c6b9caf42b1afe24437daaee904069137d8bab077be8b8" dependencies = [ - "axum-core", + "axum-core 0.5.0", "base64 0.22.1", "bytes", "form_urlencoded", @@ -734,7 +761,7 @@ dependencies = [ "hyper 1.4.1", "hyper-util", "itoa", - "matchit", + "matchit 0.8.4", "memchr", "mime", "percent-encoding", @@ -754,6 +781,26 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 1.0.1", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-core" version = "0.5.0" @@ -1299,7 +1346,7 @@ dependencies = [ "aws-config", "aws-sdk-kms", "aws-sdk-s3", - "axum", + "axum 0.8.1", "base64 0.13.1", "bytes", "camino", @@ -1337,7 +1384,7 @@ dependencies = [ "tokio-stream", "tokio-util", "tower 0.5.2", - "tower-http", + "tower-http 0.6.2", "tracing", "tracing-opentelemetry", "tracing-subscriber", @@ -1359,6 +1406,47 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console-api" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8030735ecb0d128428b64cd379809817e620a40e5001c54465b99ec5feec2857" +dependencies = [ + "futures-core", + "prost", + "prost-types", + "tonic", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6539aa9c6a4cd31f4b1c040f860a1eac9aa80e7df6b05d506a6e7179936d6a01" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures-task", + "hdrhistogram", + "humantime", + "hyper-util", + "parking_lot 0.12.1", + "prost", + "prost-types", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic", + "tonic-web", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -3435,6 +3523,12 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "matchit" version = "0.8.4" @@ -4892,6 +4986,7 @@ dependencies = [ "clap", "clashmap", "compute_api", + "console-subscriber", "consumption_metrics", "ecdsa 0.16.9", "ed25519-dalek", @@ -5404,7 +5499,7 @@ dependencies = [ "async-trait", "getrandom 0.2.11", "http 1.1.0", - "matchit", + "matchit 0.8.4", "opentelemetry", "reqwest", "reqwest-middleware", @@ -6893,6 +6988,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", + "tracing", "windows-sys 0.52.0", ] @@ -7132,9 +7228,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ + "async-stream", "async-trait", + "axum 0.7.9", "base64 0.22.1", "bytes", + "h2 0.4.4", "http 1.1.0", "http-body 1.0.0", "http-body-util", @@ -7146,6 +7245,7 @@ dependencies = [ "prost", "rustls-native-certs 0.8.0", "rustls-pemfile 2.1.1", + "socket2", "tokio", "tokio-rustls 0.26.0", "tokio-stream", @@ -7169,6 +7269,26 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "tonic-web" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5299dd20801ad736dccb4a5ea0da7376e59cd98f213bf1c3d478cf53f4834b58" +dependencies = [ + "base64 0.22.1", + "bytes", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "pin-project", + "tokio-stream", + "tonic", + "tower-http 0.5.2", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.4.13" @@ -7205,6 +7325,22 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower-http" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +dependencies = [ + "bitflags 2.8.0", + "bytes", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "pin-project-lite", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-http" version = "0.6.2" @@ -7335,6 +7471,7 @@ checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", "once_cell", + "parking_lot 0.12.1", "regex", "serde", "serde_json", @@ -7637,7 +7774,7 @@ name = "vm_monitor" version = "0.1.0" dependencies = [ "anyhow", - "axum", + "axum 0.8.1", "cgroups-rs", "clap", "futures", @@ -8126,6 +8263,7 @@ dependencies = [ "chrono", "clap", "clap_builder", + "crossbeam-utils", "crypto-bigint 0.5.5", "der 0.7.8", "deranged", @@ -8142,6 +8280,7 @@ dependencies = [ "getrandom 0.2.11", "half", "hashbrown 0.14.5", + "hdrhistogram", "hex", "hmac", "hyper 0.14.30", @@ -8197,9 +8336,11 @@ dependencies = [ "toml_edit", "tonic", "tower 0.4.13", + "tower 0.5.2", "tracing", "tracing-core", "tracing-log", + "tracing-subscriber", "url", "zerocopy", "zeroize", diff --git a/Cargo.toml b/Cargo.toml index 76b54ae1d8..3b3740facd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,6 +80,7 @@ chrono = { version = "0.4", default-features = false, features = ["clock"] } clap = { version = "4.0", features = ["derive", "env"] } clashmap = { version = "1.0", features = ["raw-api"] } comfy-table = "7.1" +console-subscriber = { version = "0.4.1", features = ["parking_lot", "grpc-web"] } const_format = "0.2" crc32c = "0.6" diatomic-waker = { version = "0.2.3" } diff --git a/Dockerfile b/Dockerfile index 7ba54c8ca5..af096a3c69 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,7 @@ COPY --chown=nonroot . . ARG ADDITIONAL_RUSTFLAGS RUN set -e \ - && RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=-Wl,--no-rosegment -Cforce-frame-pointers=yes ${ADDITIONAL_RUSTFLAGS}" cargo build \ + && RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=-Wl,--no-rosegment -Cforce-frame-pointers=yes --cfg=tokio_unstable ${ADDITIONAL_RUSTFLAGS}" cargo build \ --bin pg_sni_router \ --bin pageserver \ --bin pagectl \ diff --git a/proxy/Cargo.toml b/proxy/Cargo.toml index d7880ea7b9..3d851d0ae9 100644 --- a/proxy/Cargo.toml +++ b/proxy/Cargo.toml @@ -27,6 +27,7 @@ chrono.workspace = true clap = { workspace = true, features = ["derive", "env"] } clashmap.workspace = true compute_api.workspace = true +console-subscriber.workspace = true consumption_metrics.workspace = true env_logger.workspace = true framed-websockets.workspace = true diff --git a/proxy/src/logging.rs b/proxy/src/logging.rs index 97c9f5a59c..f1fffd1a3d 100644 --- a/proxy/src/logging.rs +++ b/proxy/src/logging.rs @@ -69,7 +69,12 @@ pub async fn init() -> anyhow::Result { None }; + let tokio_console = console_subscriber::ConsoleLayer::builder() + .enable_grpc_web(true) + .spawn(); + tracing_subscriber::registry() + .with(tokio_console) .with(env_filter) .with(otlp_layer) .with(json_log_layer) diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index 2c65401154..8e320df983 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -25,6 +25,7 @@ camino = { version = "1", default-features = false, features = ["serde1"] } chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "wasmbind"] } clap = { version = "4", features = ["derive", "env", "string"] } clap_builder = { version = "4", default-features = false, features = ["color", "env", "help", "std", "string", "suggestions", "usage"] } +crossbeam-utils = { version = "0.8" } crypto-bigint = { version = "0.5", features = ["generic-array", "zeroize"] } der = { version = "0.7", default-features = false, features = ["oid", "pem", "std"] } deranged = { version = "0.3", default-features = false, features = ["powerfmt", "serde", "std"] } @@ -40,11 +41,12 @@ generic-array = { version = "0.14", default-features = false, features = ["more_ getrandom = { version = "0.2", default-features = false, features = ["std"] } half = { version = "2", default-features = false, features = ["num-traits"] } hashbrown = { version = "0.14", features = ["raw"] } +hdrhistogram = { version = "7" } hex = { version = "0.4", features = ["serde"] } hmac = { version = "0.12", default-features = false, features = ["reset"] } hyper-582f2526e08bb6a0 = { package = "hyper", version = "0.14", features = ["full"] } hyper-dff4ba8e3ae991db = { package = "hyper", version = "1", features = ["full"] } -hyper-util = { version = "0.1", features = ["client-legacy", "http1", "http2", "server", "service"] } +hyper-util = { version = "0.1", features = ["client-legacy", "server-auto", "service"] } indexmap-dff4ba8e3ae991db = { package = "indexmap", version = "1", default-features = false, features = ["std"] } indexmap-f595c2ba2a3f28df = { package = "indexmap", version = "2", features = ["serde"] } itertools = { version = "0.12" } @@ -83,16 +85,18 @@ sync_wrapper = { version = "0.1", default-features = false, features = ["futures tikv-jemalloc-ctl = { version = "0.6", features = ["stats", "use_std"] } tikv-jemalloc-sys = { version = "0.6", features = ["profiling", "stats", "unprefixed_malloc_on_supported_platforms"] } time = { version = "0.3", features = ["macros", "serde-well-known"] } -tokio = { version = "1", features = ["full", "test-util"] } +tokio = { version = "1", features = ["full", "test-util", "tracing"] } tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "ring", "tls12"] } -tokio-stream = { version = "0.1" } +tokio-stream = { version = "0.1", features = ["net"] } tokio-util = { version = "0.7", features = ["codec", "compat", "io", "rt"] } toml_edit = { version = "0.22", features = ["serde"] } -tonic = { version = "0.12", default-features = false, features = ["codegen", "prost", "tls-roots"] } -tower = { version = "0.4", default-features = false, features = ["balance", "buffer", "limit", "util"] } +tonic = { version = "0.12", features = ["tls-roots"] } +tower-9fbad63c4bcf4a8f = { package = "tower", version = "0.4", default-features = false, features = ["balance", "buffer", "limit", "util"] } +tower-d8f496e17d97b5cb = { package = "tower", version = "0.5", default-features = false, features = ["log", "make", "util"] } tracing = { version = "0.1", features = ["log"] } tracing-core = { version = "0.1" } tracing-log = { version = "0.2" } +tracing-subscriber = { version = "0.3", default-features = false, features = ["env-filter", "fmt", "json", "parking_lot", "smallvec", "tracing-log"] } url = { version = "2", features = ["serde"] } zerocopy = { version = "0.7", features = ["derive", "simd"] } zeroize = { version = "1", features = ["derive", "serde"] }