Add request/response size metrics

libs/remote_storage/src/metrics.rs

@@ -162,6 +162,10 @@ pub(crate) fn start_measuring_requests(
 pub(crate) struct BucketMetrics {
     /// Full request duration until successful completion, error or cancellation.
     pub(crate) req_seconds: PassFailCancelledRequestTyped<Histogram>,
+    /// Byte size of requests.
+    pub(crate) req_bytes: RequestTyped<Histogram>,
+    /// Byte size of responses.
+    pub(crate) resp_bytes: RequestTyped<Histogram>,
     /// Total amount of seconds waited on queue.
     pub(crate) wait_seconds: RequestTyped<Histogram>,
 
@@ -189,6 +193,24 @@ impl Default for BucketMetrics {
             req_seconds.with_label_values(&[kind.as_str(), outcome.as_str()])
         });
 
+        let req_bytes = register_histogram_vec!(
+            "remote_storage_s3_request_bytes",
+            "Byte size of requests",
+            &["request_type"],
+        )
+        .unwrap();
+        let req_bytes =
+            RequestTyped::build_with(|kind| req_bytes.with_label_values(&[kind.as_str()]));
+
+        let resp_bytes = register_histogram_vec!(
+            "remote_storage_s3_response_bytes",
+            "Byte size of responses",
+            &["request_type"],
+        )
+        .unwrap();
+        let resp_bytes =
+            RequestTyped::build_with(|kind| resp_bytes.with_label_values(&[kind.as_str()]));
+
         let wait_seconds = register_histogram_vec!(
             "remote_storage_s3_wait_seconds",
             "Seconds rate limited",
@@ -216,6 +238,8 @@ impl Default for BucketMetrics {
 
         Self {
             req_seconds,
+            req_bytes,
+            resp_bytes,
             wait_seconds,
             cancelled_waits,
             deleted_objects_total,

libs/remote_storage/src/s3_bucket.rs

@@ -39,6 +39,7 @@ use futures_util::StreamExt;
 use hyper::body::Frame;
 use scopeguard::ScopeGuard;
 use tokio_util::sync::CancellationToken;
+use tracing::warn;
 use utils::backoff;
 
 use super::StorageMetadata;
@@ -330,6 +331,15 @@ impl S3Bucket {
             .try_into()
             .map_err(|e: ConversionError| DownloadError::Other(e.into()))?;
 
+        if let Some(size) = object_output.content_length {
+            crate::metrics::BUCKET_METRICS
+                .resp_bytes
+                .get(kind)
+                .observe(size as f64);
+        } else {
+            warn!("No Content-Length header");
+        }
+
         let body = object_output.body;
         let body = ByteStreamAsStream::from(body);
         let body = PermitCarrying::new(permit, body);
@@ -739,6 +749,10 @@ impl RemoteStorage for S3Bucket {
             crate::metrics::BUCKET_METRICS
                 .req_seconds
                 .observe_elapsed(kind, inner, started_at);
+            crate::metrics::BUCKET_METRICS
+                .req_bytes
+                .get(kind)
+                .observe(from_size_bytes as f64);
         }
 
         match res {