Improve synthetic_size json output:

- add new fields 'parent_lsn' and 'wal_from_parent' to segments; - output 'timeline_inputs' lsns as u64 to be consistent with other places
Fix missing VM cgconfig.conf (#3718 )
2026-05-17 21:20:37 +00:00 · 2023-03-02 12:53:00 +02:00 · 2023-02-28 21:11:00 -08:00 · 2023-02-27 12:44:08 +00:00
10 changed files with 83 additions and 129 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3067,15 +3067,6 @@ dependencies = [
 "workspace_hack",
 ]

-[[package]]
-name = "remove_dir_all"
-version = "0.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
-dependencies = [
- "winapi",
-]
-
 [[package]]
 name = "reqwest"
 version = "0.11.14"
@@ -3849,16 +3840,15 @@ dependencies = [

 [[package]]
 name = "tempfile"
-version = "3.3.0"
+version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4"
+checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95"
 dependencies = [
 "cfg-if",
 "fastrand",
- "libc",
 "redox_syscall",
- "remove_dir_all",
- "winapi",
+ "rustix",
+ "windows-sys 0.42.0",
 ]

 [[package]]
@@ -4544,7 +4534,6 @@ dependencies = [
 "tracing",
 "tracing-subscriber",
 "url",
- "uuid",
 "workspace_hack",
 ]

--- a/Cargo.toml
+++ b/Cargo.toml
@@ -150,7 +150,7 @@ workspace_hack = { version = "0.1", path = "./workspace_hack/" }
 criterion = "0.4"
 rcgen = "0.10"
 rstest = "0.16"
-tempfile = "3.2"
+tempfile = "3.4"
 tonic-build = "0.8"

 # This is only needed for proxy's tests.
--- a/Dockerfile.vm-compute-node
+++ b/Dockerfile.vm-compute-node
@@ -10,7 +10,6 @@ RUN set -e \
 	&& rm -f /etc/inittab \
 	&& touch /etc/inittab

-ADD vm-cgconfig.conf /etc/cgconfig.conf
 RUN set -e \
 	&& echo "::sysinit:cgconfigparser -l /etc/cgconfig.conf -s 1664" >> /etc/inittab \
 	&& echo "::respawn:su vm-informant -c '/usr/local/bin/vm-informant --auto-restart --cgroup=neon-postgres'" >> /etc/inittab
@@ -26,6 +25,7 @@ RUN apt update && \
 RUN adduser vm-informant --disabled-password --no-create-home
 USER postgres

+ADD vm-cgconfig.conf /etc/cgconfig.conf
 COPY --from=informant /etc/inittab /etc/inittab
 COPY --from=informant /usr/bin/vm-informant /usr/local/bin/vm-informant

--- a/libs/tenant_size_model/src/lib.rs
+++ b/libs/tenant_size_model/src/lib.rs
@@ -37,10 +37,15 @@ pub struct Segment {
    /// LSN at this point
    pub lsn: u64,

+    pub parent_lsn: Option<u64>,
+
+    pub wal_from_parent: Option<u64>,
+
    /// Logical size at this node, if known.
    pub size: Option<u64>,

    /// If true, the segment from parent to this node is needed by `retention_period`
+    /// if wal_from_parent is needed by `retention_period`?
    pub needed: bool,
 }

--- a/libs/tenant_size_model/tests/tests.rs
+++ b/libs/tenant_size_model/tests/tests.rs
@@ -17,6 +17,8 @@ impl ScenarioBuilder {
        let init_segment = Segment {
            parent: None,
            lsn: 0,
+            parent_lsn: None,
+            wal_from_parent: None,
            size: Some(0),
            needed: false, // determined later
        };
@@ -36,6 +38,8 @@ impl ScenarioBuilder {
        let newseg = Segment {
            parent: Some(lastseg_id),
            lsn: lastseg.lsn + lsn_bytes,
+            parent_lsn: Some(lastseg.lsn),
+            wal_from_parent: Some(lsn_bytes),
            size: Some((lastseg.size.unwrap() as i64 + size_bytes) as u64),
            needed: false,
        };
--- a/libs/utils/Cargo.toml
+++ b/libs/utils/Cargo.toml
@@ -39,7 +39,7 @@ pq_proto.workspace = true

 workspace_hack.workspace = true
 url.workspace = true
-uuid = { version = "1.2", features = ["v4", "serde"] }
+
 [dev-dependencies]
 byteorder.workspace = true
 bytes.workspace = true
--- a/libs/utils/src/http/endpoint.rs
+++ b/libs/utils/src/http/endpoint.rs
@@ -63,26 +63,8 @@ async fn prometheus_metrics_handler(_req: Request<Body>) -> Result<Response<Body
    Ok(response)
 }

-pub fn add_request_id_middleware<B: hyper::body::HttpBody + Send + Sync + 'static>(
-) -> Middleware<B, ApiError> {
-    Middleware::pre(move |mut req| async move {
-        let headers = req.headers_mut();
-        let name = HeaderName::from_str("UUID").expect("created header name");
-        let request_id = uuid::Uuid::new_v4().to_string();
-        let value = HeaderValue::from_str(&request_id).unwrap();
-        headers.insert(name, value);
-        if req.method() == Method::GET {
-            tracing::debug!("{} {} {}", req.method(), req.uri().path(), request_id);
-        } else {
-            tracing::info!("{} {} {}", req.method(), req.uri().path(), request_id);
-        }
-        Ok(req)
-    })
-}
-
 pub fn make_router() -> RouterBuilder<hyper::Body, ApiError> {
    Router::builder()
-        .middleware(add_request_id_middleware())
        .middleware(Middleware::post_with_info(logger))
        .get("/metrics", prometheus_metrics_handler)
        .err_handler(error::handler)
--- a/pageserver/src/http/routes.rs
+++ b/pageserver/src/http/routes.rs
@@ -1,14 +1,9 @@
 use std::collections::HashMap;
-use std::str::FromStr;
 use std::sync::Arc;

 use anyhow::{anyhow, Context, Result};
-use futures::Future;
-use hyper::body::HttpBody;
-use hyper::header::HeaderName;
-use hyper::http::HeaderValue;
+use hyper::StatusCode;
 use hyper::{Body, Request, Response, Uri};
-use hyper::{Method, StatusCode};
 use metrics::launch_timestamp::LaunchTimestamp;
 use pageserver_api::models::DownloadRemoteLayersTaskSpawnRequest;
 use remote_storage::GenericRemoteStorage;
@@ -1162,42 +1157,6 @@ pub fn make_router(
            "/v1/tenant/:tenant_id/timeline/:timeline_id/layer/:layer_file_name",
            evict_timeline_layer_handler,
        )
-        .get(
-            "/v1/panic",
-            wrap_span("always_panic_handler", always_panic_handler),
-        )
+        .get("/v1/panic", always_panic_handler)
        .any(handler_404))
 }
-
-fn wrap_span<H, R, B, E>(
-    handler_name: &'static str,
-    handler: H,
-) -> impl Fn(Request<hyper::Body>) -> R
-where
-    H: Fn(Request<hyper::Body>) -> R + Send + Sync + 'static,
-    B: HttpBody + Send + Sync + 'static,
-    E: Into<Box<dyn std::error::Error + Send + Sync>>,
-    R: Future<Output = Result<Response<B>, E>> + Send + 'static,
-{
-    move |r| -> R {
-        async {
-            let headers = r.headers_mut();
-            let name = HeaderName::from_str("UUID").expect("created header name");
-            let request_id = "foo";
-            let value = HeaderValue::from_str(&request_id).unwrap();
-            headers.insert(name, value);
-            if r.method() == Method::GET {
-                tracing::debug!("{} {} {}", r.method(), r.uri().path(), request_id);
-            } else {
-                tracing::info!("{} {} {}", r.method(), r.uri().path(), request_id);
-            }
-            handler(r)
-                .instrument(info_span!(
-                    "request",
-                    handler = handler_name,
-                    request_id = request_id
-                ))
-                .await
-        }
-    }
-}
--- a/pageserver/src/tenant/size.rs
+++ b/pageserver/src/tenant/size.rs
@@ -85,23 +85,16 @@ pub struct TimelineInputs {
    #[serde_as(as = "Option<serde_with::DisplayFromStr>")]
    pub ancestor_id: Option<TimelineId>,

-    #[serde_as(as = "serde_with::DisplayFromStr")]
    ancestor_lsn: Lsn,
-    #[serde_as(as = "serde_with::DisplayFromStr")]
    last_record: Lsn,
-    #[serde_as(as = "serde_with::DisplayFromStr")]
    latest_gc_cutoff: Lsn,
-    #[serde_as(as = "serde_with::DisplayFromStr")]
    horizon_cutoff: Lsn,
-    #[serde_as(as = "serde_with::DisplayFromStr")]
    pitr_cutoff: Lsn,

    /// Cutoff point based on GC settings
-    #[serde_as(as = "serde_with::DisplayFromStr")]
    next_gc_cutoff: Lsn,

    /// Cutoff point calculated from the user-supplied 'max_retention_period'
-    #[serde_as(as = "Option<serde_with::DisplayFromStr>")]
    retention_param_cutoff: Option<Lsn>,
 }

@@ -252,6 +245,8 @@ pub(super) async fn gather_inputs(
            segment: Segment {
                parent: None, // filled in later
                lsn: branch_start_lsn.0,
+                parent_lsn: None,
+                wal_from_parent: None,
                size: None, // filled in later
                needed: branch_start_needed,
            },
@@ -270,6 +265,8 @@ pub(super) async fn gather_inputs(
                segment: Segment {
                    parent: Some(parent),
                    lsn: lsn.0,
+                    parent_lsn: Some(segments[parent].segment.lsn),
+                    wal_from_parent: Some(lsn.0 - segments[parent].segment.lsn),
                    size: None,
                    needed: lsn > next_gc_cutoff,
                },
@@ -284,6 +281,8 @@ pub(super) async fn gather_inputs(
            segment: Segment {
                parent: Some(parent),
                lsn: last_record_lsn.0,
+                parent_lsn: Some(segments[parent].segment.lsn),
+                wal_from_parent: Some(last_record_lsn.0 - segments[parent].segment.lsn),
                size: None, // Filled in later, if necessary
                needed: true,
            },
@@ -613,32 +612,32 @@ fn verify_size_for_multiple_branches() {
  "timeline_inputs": [
    {
      "timeline_id": "20b129c9b50cff7213e6503a31b2a5ce",
-      "ancestor_lsn": "0/18D3D98",
-      "last_record": "0/2230CD0",
-      "latest_gc_cutoff": "0/1698C48",
-      "horizon_cutoff": "0/2210CD0",
-      "pitr_cutoff": "0/2210CD0",
-      "next_gc_cutoff": "0/2210CD0",
+      "ancestor_lsn": 26033560,
+      "last_record": 35851472,
+      "latest_gc_cutoff": 23694408,
+      "horizon_cutoff": 35720400,
+      "pitr_cutoff": 35720400,
+      "next_gc_cutoff": 35720400,
      "retention_param_cutoff": null
    },
    {
      "timeline_id": "454626700469f0a9914949b9d018e876",
-      "ancestor_lsn": "0/176D998",
-      "last_record": "0/1837770",
-      "latest_gc_cutoff": "0/1698C48",
-      "horizon_cutoff": "0/1817770",
-      "pitr_cutoff": "0/1817770",
-      "next_gc_cutoff": "0/1817770",
+      "ancestor_lsn": 24566168,
+      "last_record": 25393008,
+      "latest_gc_cutoff": 23694408,
+      "horizon_cutoff": 25261936,
+      "pitr_cutoff": 25261936,
+      "next_gc_cutoff": 25261936,
      "retention_param_cutoff": null
    },
    {
      "timeline_id": "cb5e3cbe60a4afc00d01880e1a37047f",
-      "ancestor_lsn": "0/0",
-      "last_record": "0/18D3D98",
-      "latest_gc_cutoff": "0/1698C48",
-      "horizon_cutoff": "0/18B3D98",
-      "pitr_cutoff": "0/18B3D98",
-      "next_gc_cutoff": "0/18B3D98",
+      "ancestor_lsn": 0,
+      "last_record": 26033560,
+      "latest_gc_cutoff": 23694408,
+      "horizon_cutoff":25902488,
+      "pitr_cutoff":25902488,
+      "next_gc_cutoff":25902488,
      "retention_param_cutoff": null
    }
  ]
@@ -688,13 +687,13 @@ fn verify_size_for_one_branch() {
  "timeline_inputs": [
    {
      "timeline_id": "f15ae0cf21cce2ba27e4d80c6709a6cd",
-      "ancestor_lsn": "0/0",
-      "last_record": "47/280A5860",
-      "latest_gc_cutoff": "47/240A5860",
-      "horizon_cutoff": "47/240A5860",
-      "pitr_cutoff": "47/240A5860",
-      "next_gc_cutoff": "47/240A5860",
-      "retention_param_cutoff": "0/0"
+      "ancestor_lsn": 0,
+      "last_record": 305547335776,
+      "latest_gc_cutoff": 305614444640,
+      "horizon_cutoff": 305614444640,
+      "pitr_cutoff": 305614444640,
+      "next_gc_cutoff": 305614444640,
+      "retention_param_cutoff": 0
    }
  ]
 }"#;
@@ -707,12 +706,15 @@ fn verify_size_for_one_branch() {
    println!("result: {:?}", serde_json::to_string(&res.segments));

    use utils::lsn::Lsn;
-    let latest_gc_cutoff_lsn: Lsn = "47/240A5860".parse().unwrap();
-    let last_lsn: Lsn = "47/280A5860".parse().unwrap();
+    let latest_gc_cutoff_lsn: Lsn = Lsn(305614444640);
+    let last_lsn: Lsn = Lsn(305547335776);
    println!(
-        "latest_gc_cutoff lsn 47/240A5860 is {}, last_lsn lsn 47/280A5860 is {}",
+        "latest_gc_cutoff lsn {} is {}, last_lsn lsn {} is {}",
+        latest_gc_cutoff_lsn,
        u64::from(latest_gc_cutoff_lsn),
+        last_lsn,
        u64::from(last_lsn)
    );
+
    assert_eq!(res.total_size, 220121784320);
 }
--- a/test_runner/regress/test_tenant_size.py
+++ b/test_runner/regress/test_tenant_size.py
@@ -52,12 +52,26 @@ def test_empty_tenant_size(neon_simple_env: NeonEnv, test_output_dir: Path):
    expected_inputs = {
        "segments": [
            {
-                "segment": {"parent": None, "lsn": 23694408, "size": 25362432, "needed": True},
+                "segment": {
+                    "parent": None,
+                    "lsn": 23694408,
+                    "parent_lsn": None,
+                    "wal_from_parent": None,
+                    "size": 25362432,
+                    "needed": True,
+                },
                "timeline_id": f"{main_timeline_id}",
                "kind": "BranchStart",
            },
            {
-                "segment": {"parent": 0, "lsn": 23694528, "size": None, "needed": True},
+                "segment": {
+                    "parent": 0,
+                    "lsn": 23694528,
+                    "parent_lsn": 23694408,
+                    "wal_from_parent": 120,
+                    "size": None,
+                    "needed": True,
+                },
                "timeline_id": f"{main_timeline_id}",
                "kind": "BranchEnd",
            },
@@ -66,16 +80,17 @@ def test_empty_tenant_size(neon_simple_env: NeonEnv, test_output_dir: Path):
            {
                "timeline_id": f"{main_timeline_id}",
                "ancestor_id": None,
-                "ancestor_lsn": "0/0",
-                "last_record": "0/1698CC0",
-                "latest_gc_cutoff": "0/1698C48",
-                "horizon_cutoff": "0/0",
-                "pitr_cutoff": "0/0",
-                "next_gc_cutoff": "0/0",
+                "ancestor_lsn": 0,
+                "last_record": 23694528,
+                "latest_gc_cutoff": 23694408,
+                "horizon_cutoff": 0,
+                "pitr_cutoff": 0,
+                "next_gc_cutoff": 0,
                "retention_param_cutoff": None,
            }
        ],
    }
+
    expected_inputs = mask_model_inputs(expected_inputs)
    actual_inputs = mask_model_inputs(inputs)

@@ -641,7 +656,11 @@ def mask_model_inputs(x):
    if isinstance(x, dict):
        newx = {}
        for k, v in x.items():
-            if k == "size":
+            if (
+                k in ["size", "wal_from_parent", "last_record"]
+                or k.endswith("lsn")
+                or k.endswith("cutoff")
+            ):
                if v is None or v == 0:
                    # no change
                    newx[k] = v
@@ -649,12 +668,6 @@ def mask_model_inputs(x):
                    newx[k] = "<0"
                else:
                    newx[k] = ">0"
-            elif k.endswith("lsn") or k.endswith("cutoff") or k == "last_record":
-                if v is None or v == 0 or v == "0/0":
-                    # no change
-                    newx[k] = v
-                else:
-                    newx[k] = "masked"
            else:
                newx[k] = mask_model_inputs(v)
        return newx