diff --git a/libs/remote_storage/src/metrics.rs b/libs/remote_storage/src/metrics.rs index f1aa4c433b..e4f8d8a3a1 100644 --- a/libs/remote_storage/src/metrics.rs +++ b/libs/remote_storage/src/metrics.rs @@ -162,6 +162,10 @@ pub(crate) fn start_measuring_requests( pub(crate) struct BucketMetrics { /// Full request duration until successful completion, error or cancellation. pub(crate) req_seconds: PassFailCancelledRequestTyped, + /// Byte size of requests. + pub(crate) req_bytes: RequestTyped, + /// Byte size of responses. + pub(crate) resp_bytes: RequestTyped, /// Total amount of seconds waited on queue. pub(crate) wait_seconds: RequestTyped, @@ -189,6 +193,24 @@ impl Default for BucketMetrics { req_seconds.with_label_values(&[kind.as_str(), outcome.as_str()]) }); + let req_bytes = register_histogram_vec!( + "remote_storage_s3_request_bytes", + "Byte size of requests", + &["request_type"], + ) + .unwrap(); + let req_bytes = + RequestTyped::build_with(|kind| req_bytes.with_label_values(&[kind.as_str()])); + + let resp_bytes = register_histogram_vec!( + "remote_storage_s3_response_bytes", + "Byte size of responses", + &["request_type"], + ) + .unwrap(); + let resp_bytes = + RequestTyped::build_with(|kind| resp_bytes.with_label_values(&[kind.as_str()])); + let wait_seconds = register_histogram_vec!( "remote_storage_s3_wait_seconds", "Seconds rate limited", @@ -216,6 +238,8 @@ impl Default for BucketMetrics { Self { req_seconds, + req_bytes, + resp_bytes, wait_seconds, cancelled_waits, deleted_objects_total, diff --git a/libs/remote_storage/src/s3_bucket.rs b/libs/remote_storage/src/s3_bucket.rs index cde32df402..24c9d1b71b 100644 --- a/libs/remote_storage/src/s3_bucket.rs +++ b/libs/remote_storage/src/s3_bucket.rs @@ -39,6 +39,7 @@ use futures_util::StreamExt; use hyper::body::Frame; use scopeguard::ScopeGuard; use tokio_util::sync::CancellationToken; +use tracing::warn; use utils::backoff; use super::StorageMetadata; @@ -330,6 +331,15 @@ impl S3Bucket { .try_into() .map_err(|e: ConversionError| DownloadError::Other(e.into()))?; + if let Some(size) = object_output.content_length { + crate::metrics::BUCKET_METRICS + .resp_bytes + .get(kind) + .observe(size as f64); + } else { + warn!("No Content-Length header"); + } + let body = object_output.body; let body = ByteStreamAsStream::from(body); let body = PermitCarrying::new(permit, body); @@ -739,6 +749,10 @@ impl RemoteStorage for S3Bucket { crate::metrics::BUCKET_METRICS .req_seconds .observe_elapsed(kind, inner, started_at); + crate::metrics::BUCKET_METRICS + .req_bytes + .get(kind) + .observe(from_size_bytes as f64); } match res {