update test

Cargo.lock

@@ -7116,9 +7116,9 @@ dependencies = [
 
 [[package]]
 name = "tokio"
-version = "1.43.1"
+version = "1.43.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "492a604e2fd7f814268a378409e6c92b5525d747d10db9a229723f55a417958c"
+checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e"
 dependencies = [
  "backtrace",
  "bytes",

Cargo.toml

@@ -183,7 +183,7 @@ test-context = "0.3"
 thiserror = "1.0"
 tikv-jemallocator = { version = "0.6", features = ["profiling", "stats", "unprefixed_malloc_on_supported_platforms"] }
 tikv-jemalloc-ctl = { version = "0.6", features = ["stats"] }
-tokio = { version = "1.43.1", features = ["macros"] }
+tokio = { version = "1.41", features = ["macros"] }
 tokio-epoll-uring = { git = "https://github.com/neondatabase/tokio-epoll-uring.git" , branch = "main" }
 tokio-io-timeout = "1.2.0"
 tokio-postgres-rustls = "0.12.0"

compute/compute-node.Dockerfile

@@ -1527,51 +1527,6 @@ COPY --from=pgauditlogtofile-src /ext-src/ /ext-src/
 WORKDIR /ext-src/pgauditlogtofile-src
 RUN make install USE_PGXS=1 -j $(getconf _NPROCESSORS_ONLN)
 
-#########################################################################################
-#
-# Layer "pg_rest-build"
-# compile pg_rest extension
-#
-#########################################################################################
-FROM build-deps AS pg_rest-src
-ARG PG_VERSION
-ARG PG_REST_VERSION=3.0.1
-
-# Only supported for PostgreSQL v17
-RUN if [ "${PG_VERSION:?}" != "v17" ]; then \
-        echo "pg_rest extension is only supported for PostgreSQL v17" && exit 0; \
-    fi
-
-WORKDIR /ext-src
-RUN mkdir -p pg_rest-src && cd pg_rest-src && \
-    wget https://github.com/ruslantalpa/foxfirebase/raw/main/pg_rest_pg17-${PG_REST_VERSION}_neon-debian-bookworm_aarch64.deb -O pg_rest_pg17-${PG_REST_VERSION}_aarch64.deb && \
-    wget https://github.com/ruslantalpa/foxfirebase/raw/main/pg_rest_pg17-${PG_REST_VERSION}_neon-debian-bookworm_amd64.deb -O pg_rest_pg17-${PG_REST_VERSION}_amd64.deb
-
-FROM pg-build AS pg_rest-build
-ARG PG_REST_VERSION=3.0.1
-ARG PG_VERSION
-COPY --from=pg_rest-src /ext-src/ /ext-src/
-WORKDIR /ext-src/pg_rest-src
-RUN if [ "${PG_VERSION:?}" = "v17" ]; then \
-        export ARCH=$(uname -m) && \
-        echo "ARCH: $ARCH" && \
-        # Map architecture names to package names
-        if [ "$ARCH" = "x86_64" ]; then \
-            PACKAGE_ARCH="amd64"; \
-        else \
-            PACKAGE_ARCH="$ARCH"; \
-        fi && \
-        echo "Using package architecture: $PACKAGE_ARCH" && \
-        apt-get update && \
-        apt-get install -y ./pg_rest_pg17-${PG_REST_VERSION}_${PACKAGE_ARCH}.deb && \
-        sed -i 's/superuser = false/superuser = true/g' /usr/local/pgsql/share/extension/pg_rest.control && \
-        echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_rest.control && \
-        # Clean up
-        apt-get clean && rm -rf /var/lib/apt/lists/*; \
-    else \
-        echo "pg_rest extension is only supported for PostgreSQL v17, skipping build"; \
-    fi
-
 #########################################################################################
 #
 # Layer "neon-ext-build"
@@ -1667,7 +1622,6 @@ COPY --from=pg_duckdb-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=pg_repack-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=pgaudit-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=pgauditlogtofile-build /usr/local/pgsql/ /usr/local/pgsql/
-COPY --from=pg_rest-build /usr/local/pgsql/ /usr/local/pgsql/
 
 #########################################################################################
 #
@@ -1844,6 +1798,7 @@ COPY --from=pg_repack-src /ext-src/ /ext-src/
 COPY --from=pg_repack-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY compute/patches/pg_repack.patch /ext-src
 RUN cd /ext-src/pg_repack-src && patch -p1 </ext-src/pg_repack.patch && rm -f /ext-src/pg_repack.patch
+
 COPY --chmod=755 docker-compose/run-tests.sh /run-tests.sh
 RUN apt-get update && apt-get install -y libtap-parser-sourcehandler-pgtap-perl\
    && apt clean && rm -rf /ext-src/*.tar.gz /var/lib/apt/lists/*
@@ -1976,4 +1931,3 @@ RUN mkdir /var/run/rsyslogd && \
 ENV LANG=en_US.utf8
 USER postgres
 ENTRYPOINT ["/usr/local/bin/compute_ctl"]
-

pageserver/src/consumption_metrics/metrics.rs

@@ -27,6 +27,9 @@ pub(super) enum Name {
     /// Timeline logical size
     #[serde(rename = "timeline_logical_size")]
     LogicalSize,
+    /// Timeline delta from parent
+    #[serde(rename = "timeline_changed_bytes_from_parent")]
+    ChangesFromParent,
     /// Tenant remote size
     #[serde(rename = "remote_storage_size")]
     RemoteSize,
@@ -175,6 +178,18 @@ impl MetricsKey {
         .absolute_values()
     }
 
+    const fn timeline_changed_bytes_from_parent(
+        tenant_id: TenantId,
+        timeline_id: TimelineId,
+    ) -> AbsoluteValueFactory {
+        MetricsKey {
+            tenant_id,
+            timeline_id: Some(timeline_id),
+            metric: Name::ChangesFromParent,
+        }
+        .absolute_values()
+    }
+
     /// [`Tenant::remote_size`]
     ///
     /// [`Tenant::remote_size`]: crate::tenant::Tenant::remote_size
@@ -364,6 +379,7 @@ struct TimelineSnapshot {
     loaded_at: (Lsn, SystemTime),
     last_record_lsn: Lsn,
     current_exact_logical_size: Option<u64>,
+    changed_bytes_from_parent: Option<u64>,
 }
 
 impl TimelineSnapshot {
@@ -399,10 +415,23 @@ impl TimelineSnapshot {
                 }
             };
 
+            // This is an approximation of how much data has changed on this branch vs. its ancestor: the
+            // number of bytes written to the WAL, clamped to the size of the branch.
+            let changed_bytes_from_parent = current_exact_logical_size.and_then(|size| {
+                if t.get_ancestor_lsn() == Lsn::MAX {
+                    None
+                } else {
+                    t.get_last_record_lsn()
+                        .checked_sub(t.get_ancestor_lsn())
+                        .map(|wal_bytes| std::cmp::min(wal_bytes.0, size))
+                }
+            });
+
             Ok(Some(TimelineSnapshot {
                 loaded_at,
                 last_record_lsn,
                 current_exact_logical_size,
+                changed_bytes_from_parent,
             }))
         }
     }
@@ -480,6 +509,17 @@ impl TimelineSnapshot {
                 metrics.push(factory.at(now, size));
             }
         }
+
+        {
+            let factory = MetricsKey::timeline_changed_bytes_from_parent(tenant_id, timeline_id);
+            let current_or_previous = self
+                .changed_bytes_from_parent
+                .or_else(|| cache.get(factory.key()).map(|item| item.value));
+
+            if let Some(size) = current_or_previous {
+                metrics.push(factory.at(now, size));
+            }
+        }
     }
 }
 

pageserver/src/consumption_metrics/metrics/tests.rs

@@ -18,6 +18,7 @@ fn startup_collected_timeline_metrics_before_advancing() {
         loaded_at: (disk_consistent_lsn, SystemTime::now()),
         last_record_lsn: disk_consistent_lsn,
         current_exact_logical_size: Some(0x42000),
+        changed_bytes_from_parent: Some(0x1000),
     };
 
     let now = DateTime::<Utc>::from(SystemTime::now());
@@ -33,7 +34,8 @@ fn startup_collected_timeline_metrics_before_advancing() {
                 0
             ),
             MetricsKey::written_size(tenant_id, timeline_id).at(now, disk_consistent_lsn.0),
-            MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 0x42000)
+            MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 0x42000),
+            MetricsKey::timeline_changed_bytes_from_parent(tenant_id, timeline_id).at(now, 0x1000)
         ]
     );
 }
@@ -60,6 +62,7 @@ fn startup_collected_timeline_metrics_second_round() {
         loaded_at: (disk_consistent_lsn, init),
         last_record_lsn: disk_consistent_lsn,
         current_exact_logical_size: Some(0x42000),
+        changed_bytes_from_parent: Some(0x1000),
     };
 
     snap.to_metrics(tenant_id, timeline_id, now, &mut metrics, &cache);
@@ -69,7 +72,8 @@ fn startup_collected_timeline_metrics_second_round() {
         &[
             MetricsKey::written_size_delta(tenant_id, timeline_id).from_until(before, now, 0),
             MetricsKey::written_size(tenant_id, timeline_id).at(now, disk_consistent_lsn.0),
-            MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 0x42000)
+            MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 0x42000),
+            MetricsKey::timeline_changed_bytes_from_parent(tenant_id, timeline_id).at(now, 0x1000)
         ]
     );
 }
@@ -104,6 +108,7 @@ fn startup_collected_timeline_metrics_nth_round_at_same_lsn() {
         loaded_at: (disk_consistent_lsn, init),
         last_record_lsn: disk_consistent_lsn,
         current_exact_logical_size: Some(0x42000),
+        changed_bytes_from_parent: Some(0x1000),
     };
 
     snap.to_metrics(tenant_id, timeline_id, now, &mut metrics, &cache);
@@ -113,7 +118,8 @@ fn startup_collected_timeline_metrics_nth_round_at_same_lsn() {
         &[
             MetricsKey::written_size_delta(tenant_id, timeline_id).from_until(just_before, now, 0),
             MetricsKey::written_size(tenant_id, timeline_id).at(now, disk_consistent_lsn.0),
-            MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 0x42000)
+            MetricsKey::timeline_logical_size(tenant_id, timeline_id).at(now, 0x42000),
+            MetricsKey::timeline_changed_bytes_from_parent(tenant_id, timeline_id).at(now, 0x1000)
         ]
     );
 }
@@ -141,6 +147,7 @@ fn post_restart_written_sizes_with_rolled_back_last_record_lsn() {
         loaded_at: (Lsn(50), at_restart),
         last_record_lsn: Lsn(50),
         current_exact_logical_size: None,
+        changed_bytes_from_parent: None,
     };
 
     let mut cache = HashMap::from([
@@ -202,6 +209,7 @@ fn post_restart_current_exact_logical_size_uses_cached() {
         loaded_at: (Lsn(50), at_restart),
         last_record_lsn: Lsn(50),
         current_exact_logical_size: None,
+        changed_bytes_from_parent: Some(0x1000),
     };
 
     let cache = HashMap::from([MetricsKey::timeline_logical_size(tenant_id, timeline_id)

pageserver/src/tenant/timeline/compaction.rs

@@ -1244,10 +1244,6 @@ impl Timeline {
         let mut replace_image_layers = Vec::new();
 
         for layer in layers_to_rewrite {
-            if self.cancel.is_cancelled() {
-                return Err(CompactionError::ShuttingDown);
-            }
-
             tracing::info!(layer=%layer, "Rewriting layer after shard split...");
             let mut image_layer_writer = ImageLayerWriter::new(
                 self.conf,

pgxn/neon/pagestore_client.h

@@ -274,8 +274,13 @@ typedef struct
 	XLogRecPtr effective_request_lsn;
 } neon_request_lsns;
 
+#if PG_MAJORVERSION_NUM < 16
+extern PGDLLEXPORT void neon_read_at_lsn(NRelFileInfo rnode, ForkNumber forkNum, BlockNumber blkno,
+										 neon_request_lsns request_lsns, char *buffer);
+#else
 extern PGDLLEXPORT void neon_read_at_lsn(NRelFileInfo rnode, ForkNumber forkNum, BlockNumber blkno,
 										 neon_request_lsns request_lsns, void *buffer);
+#endif
 extern int64 neon_dbsize(Oid dbNode);
 
 /* utils for neon relsize cache */

pgxn/neon/pagestore_smgr.c

@@ -1900,6 +1900,7 @@ neon_wallog_pagev(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum,
 		log_pages = true;
 	}
 	else if (XLogInsertAllowed() &&
+			 !ShutdownRequestPending &&
 			 (forknum == FSM_FORKNUM || forknum == VISIBILITYMAP_FORKNUM))
 	{
 		log_pages = true;
@@ -3153,8 +3154,13 @@ neon_writeback(SMgrRelation reln, ForkNumber forknum,
  * The offsets in request_lsns, buffers, and mask are linked.
  */
 static void
+#if PG_MAJORVERSION_NUM < 16
+neon_read_at_lsnv(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber base_blockno, neon_request_lsns *request_lsns,
+				  char **buffers, BlockNumber nblocks, const bits8 *mask)
+#else
 neon_read_at_lsnv(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber base_blockno, neon_request_lsns *request_lsns,
 				  void **buffers, BlockNumber nblocks, const bits8 *mask)
+#endif
 {
 	NeonResponse *resp;
 	uint64		ring_index;
@@ -3350,8 +3356,13 @@ Retry:
  * To avoid breaking tests in the runtime please keep function signature in sync.
  */
 void
+#if PG_MAJORVERSION_NUM < 16
+neon_read_at_lsn(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno,
+				 neon_request_lsns request_lsns, char *buffer)
+#else
 neon_read_at_lsn(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno,
 				 neon_request_lsns request_lsns, void *buffer)
+#endif
 {
 	neon_read_at_lsnv(rinfo, forkNum, blkno, &request_lsns, &buffer, 1, NULL);
 }

pgxn/neon_test_utils/neontest.c

@@ -50,8 +50,13 @@ PG_FUNCTION_INFO_V1(trigger_segfault);
  * Linkage to functions in neon module.
  * The signature here would need to be updated whenever function parameters change in pagestore_smgr.c
  */
+#if PG_MAJORVERSION_NUM < 16
+typedef void (*neon_read_at_lsn_type) (NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno,
+									   neon_request_lsns request_lsns, char *buffer);
+#else
 typedef void (*neon_read_at_lsn_type) (NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno,
 									   neon_request_lsns request_lsns, void *buffer);
+#endif
 
 static neon_read_at_lsn_type neon_read_at_lsn_ptr;
 

safekeeper/src/bin/safekeeper.rs

@@ -223,9 +223,6 @@ struct Args {
     /// Flag to use https for requests to peer's safekeeper API.
     #[arg(long)]
     pub use_https_safekeeper_api: bool,
-    /// Path to the JWT auth token used to authenticate with other safekeepers.
-    #[arg(long)]
-    auth_token_path: Option<Utf8PathBuf>,
 }
 
 // Like PathBufValueParser, but allows empty string.
@@ -344,24 +341,14 @@ async fn main() -> anyhow::Result<()> {
     };
 
     // Load JWT auth token to connect to other safekeepers for pull_timeline.
-    // First check if the env var is present, then check the arg with the path.
-    // We want to deprecate and remove the env var method in the future.
     let sk_auth_token = match var("SAFEKEEPER_AUTH_TOKEN") {
         Ok(v) => {
             info!("loaded JWT token for authentication with safekeepers");
             Some(SecretString::from(v))
         }
         Err(VarError::NotPresent) => {
-            if let Some(auth_token_path) = args.auth_token_path.as_ref() {
-                info!(
-                    "loading JWT token for authentication with safekeepers from {auth_token_path}"
-                );
-                let auth_token = tokio::fs::read_to_string(auth_token_path).await?;
-                Some(SecretString::from(auth_token.trim().to_owned()))
-            } else {
-                info!("no JWT token for authentication with safekeepers detected");
-                None
-            }
+            info!("no JWT token for authentication with safekeepers detected");
+            None
         }
         Err(_) => {
             warn!("JWT token for authentication with safekeepers is not unicode");

storage_controller/src/http.rs

@@ -899,7 +899,7 @@ async fn handle_node_status(req: Request<Body>) -> Result<Response<Body>, ApiErr
     let state = get_state(&req);
     let node_id: NodeId = parse_request_param(&req, "node_id")?;
 
-    let node_status = state.service.get_node(node_id).await?.describe();
+    let node_status = state.service.get_node(node_id).await?;
 
     json_response(StatusCode::OK, node_status)
 }

test_runner/regress/test_pageserver_metric_collection.py

@@ -511,4 +511,5 @@ PER_METRIC_VERIFIERS = {
     "written_data_bytes_delta": WrittenDataDeltaVerifier,
     "timeline_logical_size": CannotVerifyAnything,
     "synthetic_storage_size": SyntheticSizeVerifier,
+    "timeline_changed_bytes_from_parent": CannotVerifyAnything,
 }

test_runner/regress/test_storage_controller.py

@@ -4109,7 +4109,6 @@ def test_storcon_create_delete_sk_down(neon_env_builder: NeonEnvBuilder, restart
     env.storage_controller.allowed_errors.extend(
         [
             ".*Call to safekeeper.* management API still failed after.*",
-            ".*Call to safekeeper.* management API failed, will retry.*",
             ".*reconcile_one.*tenant_id={tenant_id}.*Call to safekeeper.* management API still failed after.*",
         ]
     )

vendor/revisions.json

@@ -1,18 +1,18 @@
 {
   "v17": [
     "17.4",
-    "66114c23bc61205b0e3fb1e77ee76a4abc1eb4b8"
+    "c9e4ff5a38907acd71107634055bf2609aba43a5"
   ],
   "v16": [
     "16.8",
-    "d56e79cd5d6136c159b1d8d98acb7981d4b69364"
+    "746bd9ffe5c29bce030eaea1031054057f3c5d45"
   ],
   "v15": [
     "15.12",
-    "aeb292eeace9072e07071254b6ffc7a74007d4d2"
+    "23708b3aca9adf163aa0973eb63d9afc0e4a04c3"
   ],
   "v14": [
     "14.17",
-    "a0391901a2af13aa029b905272a5b2024133c926"
+    "8cca70c22e2894dd4645f9a940086ac437b0a11b"
   ]
 }