Improve synthetic_size json output:

- add new fields 'parent_lsn' and 'wal_from_parent' to segments;
- output 'timeline_inputs' lsns as u64 to be consistent with other places

Cargo.lock

@@ -3067,15 +3067,6 @@ dependencies = [
  "workspace_hack",
 ]
 
-[[package]]
-name = "remove_dir_all"
-version = "0.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
-dependencies = [
- "winapi",
-]
-
 [[package]]
 name = "reqwest"
 version = "0.11.14"
@@ -3849,16 +3840,15 @@ dependencies = [
 
 [[package]]
 name = "tempfile"
-version = "3.3.0"
+version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4"
+checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95"
 dependencies = [
  "cfg-if",
  "fastrand",
- "libc",
  "redox_syscall",
- "remove_dir_all",
- "winapi",
+ "rustix",
+ "windows-sys 0.42.0",
 ]
 
 [[package]]

Cargo.toml

@@ -150,7 +150,7 @@ workspace_hack = { version = "0.1", path = "./workspace_hack/" }
 criterion = "0.4"
 rcgen = "0.10"
 rstest = "0.16"
-tempfile = "3.2"
+tempfile = "3.4"
 tonic-build = "0.8"
 
 # This is only needed for proxy's tests.

Dockerfile.vm-compute-node

@@ -10,7 +10,6 @@ RUN set -e \
 	&& rm -f /etc/inittab \
 	&& touch /etc/inittab
 
-ADD vm-cgconfig.conf /etc/cgconfig.conf
 RUN set -e \
 	&& echo "::sysinit:cgconfigparser -l /etc/cgconfig.conf -s 1664" >> /etc/inittab \
 	&& echo "::respawn:su vm-informant -c '/usr/local/bin/vm-informant --auto-restart --cgroup=neon-postgres'" >> /etc/inittab
@@ -26,6 +25,7 @@ RUN apt update && \
 RUN adduser vm-informant --disabled-password --no-create-home
 USER postgres
 
+ADD vm-cgconfig.conf /etc/cgconfig.conf
 COPY --from=informant /etc/inittab /etc/inittab
 COPY --from=informant /usr/bin/vm-informant /usr/local/bin/vm-informant
 

libs/pageserver_api/src/reltag.rs

@@ -98,6 +98,15 @@ impl RelTag {
 
         name
     }
+
+    pub fn with_forknum(&self, forknum: u8) -> Self {
+        RelTag {
+            forknum,
+            spcnode: self.spcnode,
+            dbnode: self.dbnode,
+            relnode: self.relnode,
+        }
+    }
 }
 
 ///

libs/tenant_size_model/src/lib.rs

@@ -37,10 +37,15 @@ pub struct Segment {
     /// LSN at this point
     pub lsn: u64,
 
+    pub parent_lsn: Option<u64>,
+
+    pub wal_from_parent: Option<u64>,
+
     /// Logical size at this node, if known.
     pub size: Option<u64>,
 
     /// If true, the segment from parent to this node is needed by `retention_period`
+    /// if wal_from_parent is needed by `retention_period`?
     pub needed: bool,
 }
 

libs/tenant_size_model/tests/tests.rs

@@ -17,6 +17,8 @@ impl ScenarioBuilder {
         let init_segment = Segment {
             parent: None,
             lsn: 0,
+            parent_lsn: None,
+            wal_from_parent: None,
             size: Some(0),
             needed: false, // determined later
         };
@@ -36,6 +38,8 @@ impl ScenarioBuilder {
         let newseg = Segment {
             parent: Some(lastseg_id),
             lsn: lastseg.lsn + lsn_bytes,
+            parent_lsn: Some(lastseg.lsn),
+            wal_from_parent: Some(lsn_bytes),
             size: Some((lastseg.size.unwrap() as i64 + size_bytes) as u64),
             needed: false,
         };

pageserver/src/basebackup.rs

@@ -33,6 +33,7 @@ use pageserver_api::reltag::{RelTag, SlruKind};
 
 use postgres_ffi::pg_constants::{DEFAULTTABLESPACE_OID, GLOBALTABLESPACE_OID};
 use postgres_ffi::pg_constants::{PGDATA_SPECIAL_FILES, PGDATA_SUBDIRS, PG_HBA};
+use postgres_ffi::relfile_utils::{INIT_FORKNUM, MAIN_FORKNUM};
 use postgres_ffi::TransactionId;
 use postgres_ffi::XLogFileName;
 use postgres_ffi::PG_TLI;
@@ -190,14 +191,31 @@ where
         {
             self.add_dbdir(spcnode, dbnode, has_relmap_file).await?;
 
-            // Gather and send relational files in each database if full backup is requested.
-            if self.full_backup {
-                for rel in self
-                    .timeline
-                    .list_rels(spcnode, dbnode, self.lsn, self.ctx)
-                    .await?
-                {
-                    self.add_rel(rel).await?;
+            // If full backup is requested, include all relation files.
+            // Otherwise only include init forks of unlogged relations.
+            let rels = self
+                .timeline
+                .list_rels(spcnode, dbnode, self.lsn, self.ctx)
+                .await?;
+            for &rel in rels.iter() {
+                // Send init fork as main fork to provide well formed empty
+                // contents of UNLOGGED relations. Postgres copies it in
+                // `reinit.c` during recovery.
+                if rel.forknum == INIT_FORKNUM {
+                    // I doubt we need _init fork itself, but having it at least
+                    // serves as a marker relation is unlogged.
+                    self.add_rel(rel, rel).await?;
+                    self.add_rel(rel, rel.with_forknum(MAIN_FORKNUM)).await?;
+                    continue;
+                }
+
+                if self.full_backup {
+                    if rel.forknum == MAIN_FORKNUM && rels.contains(&rel.with_forknum(INIT_FORKNUM))
+                    {
+                        // skip this, will include it when we reach the init fork
+                        continue;
+                    }
+                    self.add_rel(rel, rel).await?;
                 }
             }
         }
@@ -220,15 +238,16 @@ where
         Ok(())
     }
 
-    async fn add_rel(&mut self, tag: RelTag) -> anyhow::Result<()> {
+    /// Add contents of relfilenode `src`, naming it as `dst`.
+    async fn add_rel(&mut self, src: RelTag, dst: RelTag) -> anyhow::Result<()> {
         let nblocks = self
             .timeline
-            .get_rel_size(tag, self.lsn, false, self.ctx)
+            .get_rel_size(src, self.lsn, false, self.ctx)
             .await?;
 
         // If the relation is empty, create an empty file
         if nblocks == 0 {
-            let file_name = tag.to_segfile_name(0);
+            let file_name = dst.to_segfile_name(0);
             let header = new_tar_header(&file_name, 0)?;
             self.ar.append(&header, &mut io::empty()).await?;
             return Ok(());
@@ -244,12 +263,12 @@ where
             for blknum in startblk..endblk {
                 let img = self
                     .timeline
-                    .get_rel_page_at_lsn(tag, blknum, self.lsn, false, self.ctx)
+                    .get_rel_page_at_lsn(src, blknum, self.lsn, false, self.ctx)
                     .await?;
                 segment_data.extend_from_slice(&img[..]);
             }
 
-            let file_name = tag.to_segfile_name(seg as u32);
+            let file_name = dst.to_segfile_name(seg as u32);
             let header = new_tar_header(&file_name, segment_data.len() as u64)?;
             self.ar.append(&header, segment_data.as_slice()).await?;
 

pageserver/src/tenant/size.rs

@@ -85,23 +85,16 @@ pub struct TimelineInputs {
     #[serde_as(as = "Option<serde_with::DisplayFromStr>")]
     pub ancestor_id: Option<TimelineId>,
 
-    #[serde_as(as = "serde_with::DisplayFromStr")]
     ancestor_lsn: Lsn,
-    #[serde_as(as = "serde_with::DisplayFromStr")]
     last_record: Lsn,
-    #[serde_as(as = "serde_with::DisplayFromStr")]
     latest_gc_cutoff: Lsn,
-    #[serde_as(as = "serde_with::DisplayFromStr")]
     horizon_cutoff: Lsn,
-    #[serde_as(as = "serde_with::DisplayFromStr")]
     pitr_cutoff: Lsn,
 
     /// Cutoff point based on GC settings
-    #[serde_as(as = "serde_with::DisplayFromStr")]
     next_gc_cutoff: Lsn,
 
     /// Cutoff point calculated from the user-supplied 'max_retention_period'
-    #[serde_as(as = "Option<serde_with::DisplayFromStr>")]
     retention_param_cutoff: Option<Lsn>,
 }
 
@@ -252,6 +245,8 @@ pub(super) async fn gather_inputs(
             segment: Segment {
                 parent: None, // filled in later
                 lsn: branch_start_lsn.0,
+                parent_lsn: None,
+                wal_from_parent: None,
                 size: None, // filled in later
                 needed: branch_start_needed,
             },
@@ -270,6 +265,8 @@ pub(super) async fn gather_inputs(
                 segment: Segment {
                     parent: Some(parent),
                     lsn: lsn.0,
+                    parent_lsn: Some(segments[parent].segment.lsn),
+                    wal_from_parent: Some(lsn.0 - segments[parent].segment.lsn),
                     size: None,
                     needed: lsn > next_gc_cutoff,
                 },
@@ -284,6 +281,8 @@ pub(super) async fn gather_inputs(
             segment: Segment {
                 parent: Some(parent),
                 lsn: last_record_lsn.0,
+                parent_lsn: Some(segments[parent].segment.lsn),
+                wal_from_parent: Some(last_record_lsn.0 - segments[parent].segment.lsn),
                 size: None, // Filled in later, if necessary
                 needed: true,
             },
@@ -613,32 +612,32 @@ fn verify_size_for_multiple_branches() {
   "timeline_inputs": [
     {
       "timeline_id": "20b129c9b50cff7213e6503a31b2a5ce",
-      "ancestor_lsn": "0/18D3D98",
-      "last_record": "0/2230CD0",
-      "latest_gc_cutoff": "0/1698C48",
-      "horizon_cutoff": "0/2210CD0",
-      "pitr_cutoff": "0/2210CD0",
-      "next_gc_cutoff": "0/2210CD0",
+      "ancestor_lsn": 26033560,
+      "last_record": 35851472,
+      "latest_gc_cutoff": 23694408,
+      "horizon_cutoff": 35720400,
+      "pitr_cutoff": 35720400,
+      "next_gc_cutoff": 35720400,
       "retention_param_cutoff": null
     },
     {
       "timeline_id": "454626700469f0a9914949b9d018e876",
-      "ancestor_lsn": "0/176D998",
-      "last_record": "0/1837770",
-      "latest_gc_cutoff": "0/1698C48",
-      "horizon_cutoff": "0/1817770",
-      "pitr_cutoff": "0/1817770",
-      "next_gc_cutoff": "0/1817770",
+      "ancestor_lsn": 24566168,
+      "last_record": 25393008,
+      "latest_gc_cutoff": 23694408,
+      "horizon_cutoff": 25261936,
+      "pitr_cutoff": 25261936,
+      "next_gc_cutoff": 25261936,
       "retention_param_cutoff": null
     },
     {
       "timeline_id": "cb5e3cbe60a4afc00d01880e1a37047f",
-      "ancestor_lsn": "0/0",
-      "last_record": "0/18D3D98",
-      "latest_gc_cutoff": "0/1698C48",
-      "horizon_cutoff": "0/18B3D98",
-      "pitr_cutoff": "0/18B3D98",
-      "next_gc_cutoff": "0/18B3D98",
+      "ancestor_lsn": 0,
+      "last_record": 26033560,
+      "latest_gc_cutoff": 23694408,
+      "horizon_cutoff":25902488,
+      "pitr_cutoff":25902488,
+      "next_gc_cutoff":25902488,
       "retention_param_cutoff": null
     }
   ]
@@ -688,13 +687,13 @@ fn verify_size_for_one_branch() {
   "timeline_inputs": [
     {
       "timeline_id": "f15ae0cf21cce2ba27e4d80c6709a6cd",
-      "ancestor_lsn": "0/0",
-      "last_record": "47/280A5860",
-      "latest_gc_cutoff": "47/240A5860",
-      "horizon_cutoff": "47/240A5860",
-      "pitr_cutoff": "47/240A5860",
-      "next_gc_cutoff": "47/240A5860",
-      "retention_param_cutoff": "0/0"
+      "ancestor_lsn": 0,
+      "last_record": 305547335776,
+      "latest_gc_cutoff": 305614444640,
+      "horizon_cutoff": 305614444640,
+      "pitr_cutoff": 305614444640,
+      "next_gc_cutoff": 305614444640,
+      "retention_param_cutoff": 0
     }
   ]
 }"#;
@@ -707,12 +706,15 @@ fn verify_size_for_one_branch() {
     println!("result: {:?}", serde_json::to_string(&res.segments));
 
     use utils::lsn::Lsn;
-    let latest_gc_cutoff_lsn: Lsn = "47/240A5860".parse().unwrap();
-    let last_lsn: Lsn = "47/280A5860".parse().unwrap();
+    let latest_gc_cutoff_lsn: Lsn = Lsn(305614444640);
+    let last_lsn: Lsn = Lsn(305547335776);
     println!(
-        "latest_gc_cutoff lsn 47/240A5860 is {}, last_lsn lsn 47/280A5860 is {}",
+        "latest_gc_cutoff lsn {} is {}, last_lsn lsn {} is {}",
+        latest_gc_cutoff_lsn,
         u64::from(latest_gc_cutoff_lsn),
+        last_lsn,
         u64::from(last_lsn)
     );
+
     assert_eq!(res.total_size, 220121784320);
 }

pageserver/src/walingest.rs

@@ -37,7 +37,7 @@ use crate::walrecord::*;
 use crate::ZERO_PAGE;
 use pageserver_api::reltag::{RelTag, SlruKind};
 use postgres_ffi::pg_constants;
-use postgres_ffi::relfile_utils::{FSM_FORKNUM, MAIN_FORKNUM, VISIBILITYMAP_FORKNUM};
+use postgres_ffi::relfile_utils::{FSM_FORKNUM, INIT_FORKNUM, MAIN_FORKNUM, VISIBILITYMAP_FORKNUM};
 use postgres_ffi::v14::nonrelfile_utils::mx_offset_to_member_segment;
 use postgres_ffi::v14::xlog_utils::*;
 use postgres_ffi::v14::CheckPoint;
@@ -762,7 +762,7 @@ impl<'a> WalIngest<'a> {
         )?;
 
         for xnode in &parsed.xnodes {
-            for forknum in MAIN_FORKNUM..=VISIBILITYMAP_FORKNUM {
+            for forknum in MAIN_FORKNUM..=INIT_FORKNUM {
                 let rel = RelTag {
                     forknum,
                     spcnode: xnode.spcnode,

test_runner/regress/test_tenant_size.py

@@ -52,12 +52,26 @@ def test_empty_tenant_size(neon_simple_env: NeonEnv, test_output_dir: Path):
     expected_inputs = {
         "segments": [
             {
-                "segment": {"parent": None, "lsn": 23694408, "size": 25362432, "needed": True},
+                "segment": {
+                    "parent": None,
+                    "lsn": 23694408,
+                    "parent_lsn": None,
+                    "wal_from_parent": None,
+                    "size": 25362432,
+                    "needed": True,
+                },
                 "timeline_id": f"{main_timeline_id}",
                 "kind": "BranchStart",
             },
             {
-                "segment": {"parent": 0, "lsn": 23694528, "size": None, "needed": True},
+                "segment": {
+                    "parent": 0,
+                    "lsn": 23694528,
+                    "parent_lsn": 23694408,
+                    "wal_from_parent": 120,
+                    "size": None,
+                    "needed": True,
+                },
                 "timeline_id": f"{main_timeline_id}",
                 "kind": "BranchEnd",
             },
@@ -66,16 +80,17 @@ def test_empty_tenant_size(neon_simple_env: NeonEnv, test_output_dir: Path):
             {
                 "timeline_id": f"{main_timeline_id}",
                 "ancestor_id": None,
-                "ancestor_lsn": "0/0",
-                "last_record": "0/1698CC0",
-                "latest_gc_cutoff": "0/1698C48",
-                "horizon_cutoff": "0/0",
-                "pitr_cutoff": "0/0",
-                "next_gc_cutoff": "0/0",
+                "ancestor_lsn": 0,
+                "last_record": 23694528,
+                "latest_gc_cutoff": 23694408,
+                "horizon_cutoff": 0,
+                "pitr_cutoff": 0,
+                "next_gc_cutoff": 0,
                 "retention_param_cutoff": None,
             }
         ],
     }
+
     expected_inputs = mask_model_inputs(expected_inputs)
     actual_inputs = mask_model_inputs(inputs)
 
@@ -641,7 +656,11 @@ def mask_model_inputs(x):
     if isinstance(x, dict):
         newx = {}
         for k, v in x.items():
-            if k == "size":
+            if (
+                k in ["size", "wal_from_parent", "last_record"]
+                or k.endswith("lsn")
+                or k.endswith("cutoff")
+            ):
                 if v is None or v == 0:
                     # no change
                     newx[k] = v
@@ -649,12 +668,6 @@ def mask_model_inputs(x):
                     newx[k] = "<0"
                 else:
                     newx[k] = ">0"
-            elif k.endswith("lsn") or k.endswith("cutoff") or k == "last_record":
-                if v is None or v == 0 or v == "0/0":
-                    # no change
-                    newx[k] = v
-                else:
-                    newx[k] = "masked"
             else:
                 newx[k] = mask_model_inputs(v)
         return newx

test_runner/regress/test_unlogged.py

@@ -0,0 +1,34 @@
+from fixtures.neon_fixtures import NeonEnv, fork_at_current_lsn
+
+
+#
+# Test UNLOGGED tables/relations. Postgres copies init fork contents to main
+# fork to reset them during recovery. In Neon, pageserver directly sends init
+# fork contents as main fork during basebackup.
+#
+def test_unlogged(neon_simple_env: NeonEnv):
+    env = neon_simple_env
+    env.neon_cli.create_branch("test_unlogged", "empty")
+    pg = env.postgres.create_start("test_unlogged")
+
+    conn = pg.connect()
+    cur = conn.cursor()
+
+    cur.execute("CREATE UNLOGGED TABLE iut (id int);")
+    # create index to test unlogged index relation as well
+    cur.execute("CREATE UNIQUE INDEX iut_idx ON iut (id);")
+    cur.execute("INSERT INTO iut values (42);")
+
+    # create another compute to fetch inital empty contents from pageserver
+    fork_at_current_lsn(env, pg, "test_unlogged_basebackup", "test_unlogged")
+    pg2 = env.postgres.create_start(
+        "test_unlogged_basebackup",
+    )
+
+    conn2 = pg2.connect()
+    cur2 = conn2.cursor()
+    # after restart table should be empty but valid
+    cur2.execute("PREPARE iut_plan (int) AS INSERT INTO iut VALUES ($1)")
+    cur2.execute("EXECUTE iut_plan (43);")
+    cur2.execute("SELECT * FROM iut")
+    assert cur2.fetchall() == [(43,)]