From e8a5e0b0eda8a49c6cfc14f37ea280dd9921d0ed Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Fri, 19 Apr 2024 14:54:22 +0100 Subject: [PATCH] add tokio-console --- .github/workflows/build_and_test.yml | 1 + Cargo.lock | 126 ++++++++++++++++++++++++--- proxy/Cargo.toml | 4 +- proxy/src/logging.rs | 5 ++ workspace_hack/Cargo.toml | 4 +- 5 files changed, 125 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 1d35fa9223..df31adb7fa 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -214,6 +214,7 @@ jobs: BUILD_TYPE: ${{ matrix.build_type }} GIT_VERSION: ${{ github.event.pull_request.head.sha || github.sha }} BUILD_TAG: ${{ needs.tag.outputs.build-tag }} + RUSTFLAGS: "--cfg=tokio_unstable" steps: - name: Fix git ownership diff --git a/Cargo.lock b/Cargo.lock index d961071872..76ddf62191 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1240,6 +1240,43 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console-api" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd326812b3fd01da5bb1af7d340d0d555fd3d4b641e7f1dfcf5962a902952787" +dependencies = [ + "futures-core", + "prost 0.12.4", + "prost-types 0.12.4", + "tonic 0.10.2", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7481d4c57092cd1c19dd541b92bdce883de840df30aa5d03fd48a3935c01842e" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures-task", + "hdrhistogram", + "humantime", + "prost-types 0.12.4", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic 0.10.2", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "const-oid" version = "0.9.5" @@ -3408,11 +3445,11 @@ dependencies = [ "opentelemetry-semantic-conventions", "opentelemetry_api", "opentelemetry_sdk", - "prost", + "prost 0.11.9", "reqwest", "thiserror", "tokio", - "tonic", + "tonic 0.9.2", ] [[package]] @@ -3423,8 +3460,8 @@ checksum = "b1e3f814aa9f8c905d0ee4bde026afd3b2577a97c10e1699912e3e44f0c4cbeb" dependencies = [ "opentelemetry_api", "opentelemetry_sdk", - "prost", - "tonic", + "prost 0.11.9", + "tonic 0.9.2", ] [[package]] @@ -4223,7 +4260,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.11.9", +] + +[[package]] +name = "prost" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0f5d036824e4761737860779c906171497f6d55681139d8312388f8fe398922" +dependencies = [ + "bytes", + "prost-derive 0.12.4", ] [[package]] @@ -4240,8 +4287,8 @@ dependencies = [ "multimap", "petgraph", "prettyplease 0.1.25", - "prost", - "prost-types", + "prost 0.11.9", + "prost-types 0.11.9", "regex", "syn 1.0.109", "tempfile", @@ -4261,13 +4308,35 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "prost-derive" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19de2de2a00075bf566bee3bd4db014b11587e84184d3f7a791bc17f1a8e9e48" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.52", +] + [[package]] name = "prost-types" version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" dependencies = [ - "prost", + "prost 0.11.9", +] + +[[package]] +name = "prost-types" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3235c33eb02c1f1e212abdbe34c78b264b038fb58ca612664343271e36e55ffe" +dependencies = [ + "prost 0.12.4", ] [[package]] @@ -4289,6 +4358,7 @@ dependencies = [ "camino-tempfile", "chrono", "clap", + "console-subscriber", "consumption_metrics", "dashmap", "env_logger", @@ -5705,10 +5775,10 @@ dependencies = [ "metrics", "once_cell", "parking_lot 0.12.1", - "prost", + "prost 0.11.9", "tokio", "tokio-stream", - "tonic", + "tonic 0.9.2", "tonic-build", "tracing", "utils", @@ -6100,6 +6170,7 @@ dependencies = [ "signal-hook-registry", "socket2 0.5.5", "tokio-macros", + "tracing", "windows-sys 0.48.0", ] @@ -6330,7 +6401,7 @@ dependencies = [ "hyper-timeout", "percent-encoding", "pin-project", - "prost", + "prost 0.11.9", "rustls-native-certs 0.6.2", "rustls-pemfile 1.0.2", "tokio", @@ -6342,6 +6413,33 @@ dependencies = [ "tracing", ] +[[package]] +name = "tonic" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.21.1", + "bytes", + "h2 0.3.26", + "http 0.2.9", + "http-body 0.4.5", + "hyper 0.14.26", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost 0.12.4", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tonic-build" version = "0.9.2" @@ -7322,6 +7420,7 @@ dependencies = [ "futures-util", "getrandom 0.2.11", "hashbrown 0.14.0", + "hdrhistogram", "hex", "hmac", "hyper 0.14.26", @@ -7336,7 +7435,7 @@ dependencies = [ "num-traits", "once_cell", "parquet", - "prost", + "prost 0.11.9", "rand 0.8.5", "regex", "regex-automata 0.4.3", @@ -7355,10 +7454,11 @@ dependencies = [ "time-macros", "tokio", "tokio-rustls 0.24.0", + "tokio-stream", "tokio-util", "toml_datetime", "toml_edit", - "tonic", + "tonic 0.9.2", "tower", "tracing", "tracing-core", diff --git a/proxy/Cargo.toml b/proxy/Cargo.toml index 1ad014878a..7f025ac4a5 100644 --- a/proxy/Cargo.toml +++ b/proxy/Cargo.toml @@ -85,7 +85,7 @@ tokio-postgres.workspace = true tokio-postgres-rustls.workspace = true tokio-rustls.workspace = true tokio-util.workspace = true -tokio = { workspace = true, features = ["signal"] } +tokio = { workspace = true, features = ["signal", "tracing"] } tracing-opentelemetry.workspace = true tracing-subscriber.workspace = true tracing-utils.workspace = true @@ -99,6 +99,8 @@ x509-parser.workspace = true postgres-protocol.workspace = true redis.workspace = true +console-subscriber = "0.2.0" + workspace_hack.workspace = true [dev-dependencies] diff --git a/proxy/src/logging.rs b/proxy/src/logging.rs index 3405b8cbc6..c987cbbe84 100644 --- a/proxy/src/logging.rs +++ b/proxy/src/logging.rs @@ -26,7 +26,12 @@ pub async fn init() -> anyhow::Result { .await .map(OpenTelemetryLayer::new); + // spawn the console server in the background, + // returning a `Layer`: + let console_layer = console_subscriber::spawn(); + tracing_subscriber::registry() + .with(console_layer) .with(env_filter) .with(otlp_layer) .with(fmt_layer) diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index d6e2cc2996..6f318481de 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -38,6 +38,7 @@ futures-sink = { version = "0.3" } futures-util = { version = "0.3", features = ["channel", "io", "sink"] } getrandom = { version = "0.2", default-features = false, features = ["std"] } hashbrown = { version = "0.14", features = ["raw"] } +hdrhistogram = { version = "7" } hex = { version = "0.4", features = ["serde"] } hmac = { version = "0.12", default-features = false, features = ["reset"] } hyper = { version = "0.14", features = ["full"] } @@ -66,8 +67,9 @@ sha2 = { version = "0.10", features = ["asm"] } smallvec = { version = "1", default-features = false, features = ["const_new", "write"] } subtle = { version = "2" } time = { version = "0.3", features = ["local-offset", "macros", "serde-well-known"] } -tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "test-util"] } +tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "test-util", "tracing"] } tokio-rustls = { version = "0.24" } +tokio-stream = { version = "0.1", features = ["net"] } tokio-util = { version = "0.7", features = ["codec", "compat", "io", "rt"] } toml_datetime = { version = "0.6", default-features = false, features = ["serde"] } toml_edit = { version = "0.19", features = ["serde"] }