Try to enable a custom postgres_exporter query.

## Problem

We don't know the number of users with the different kind of
authentication: ["sni", "endpoint in options" (A and B from
[here](https://neon.tech/docs/connect/connection-errors)),
"password_hack"]

## Summary of changes

Collect metrics by sni kind.

.cargo/config.toml

@@ -1,3 +1,17 @@
+# The binaries are really slow, if you compile them in 'dev' mode with the defaults.
+# Enable some optimizations even in 'dev' mode, to make tests faster. The basic
+# optimizations enabled by "opt-level=1" don't affect debuggability too much.
+#
+# See https://www.reddit.com/r/rust/comments/gvrgca/this_is_a_neat_trick_for_getting_good_runtime/
+#
+[profile.dev.package."*"]
+# Set the default for dependencies in Development mode.
+opt-level = 3
+
+[profile.dev]
+# Turn on a small amount of optimization in Development mode.
+opt-level = 1
+
 [build]
 # This is only present for local builds, as it will be overridden
 # by the RUSTDOCFLAGS env var in CI.

.github/workflows/build_and_test.yml

@@ -852,7 +852,7 @@ jobs:
       run:
         shell: sh -eu {0}
     env:
-      VM_BUILDER_VERSION: v0.18.5
+      VM_BUILDER_VERSION: v0.19.0
 
     steps:
       - name: Checkout
@@ -874,8 +874,7 @@ jobs:
       - name: Build vm image
         run: |
           ./vm-builder \
-            -enable-file-cache \
-            -cgroup-uid=postgres \
+            -spec=vm-image-spec.yaml \
             -src=369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} \
             -dst=369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}}
 

Dockerfile.compute-node

@@ -714,6 +714,23 @@ RUN wget https://github.com/pksunkara/pgx_ulid/archive/refs/tags/v0.1.3.tar.gz -
     cargo pgrx install --release && \
     echo "trusted = true" >> /usr/local/pgsql/share/extension/ulid.control
 
+#########################################################################################
+#
+# Layer "pg-wait-sampling-pg-build"
+# compile pg_wait_sampling extension
+#
+#########################################################################################
+FROM build-deps AS pg-wait-sampling-pg-build
+COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
+
+ENV PATH "/usr/local/pgsql/bin/:$PATH"
+RUN wget https://github.com/postgrespro/pg_wait_sampling/archive/refs/tags/v1.1.5.tar.gz -O pg_wait_sampling.tar.gz && \
+    echo 'a03da6a413f5652ce470a3635ed6ebba528c74cb26aa4cfced8aff8a8441f81ec6dd657ff62cd6ce96a4e6ce02cad9f2519ae9525367ece60497aa20faafde5c  pg_wait_sampling.tar.gz' | sha512sum -c && \
+    mkdir pg_wait_sampling-src && cd pg_wait_sampling-src && tar xvzf ../pg_wait_sampling.tar.gz --strip-components=1 -C . && \
+    make USE_PGXS=1 -j $(getconf _NPROCESSORS_ONLN) && \
+    make USE_PGXS=1 -j $(getconf _NPROCESSORS_ONLN) install && \
+    echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_wait_sampling.control
+
 #########################################################################################
 #
 # Layer "neon-pg-ext-build"
@@ -750,6 +767,7 @@ COPY --from=rdkit-pg-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=pg-uuidv7-pg-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=pg-roaringbitmap-pg-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=pg-embedding-pg-build /usr/local/pgsql/ /usr/local/pgsql/
+COPY --from=pg-wait-sampling-pg-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY pgxn/ pgxn/
 
 RUN make -j $(getconf _NPROCESSORS_ONLN) \

pageserver/src/keyspace.rs

@@ -1,7 +1,4 @@
-use crate::{
-    pgdatadir_mapping::{BASEBACKUP_CUT, METADATA_CUT},
-    repository::{key_range_size, singleton_range, Key},
-};
+use crate::repository::{key_range_size, singleton_range, Key};
 use postgres_ffi::BLCKSZ;
 use std::ops::Range;
 
@@ -25,22 +22,13 @@ impl KeySpace {
         let target_nblocks = (target_size / BLCKSZ as u64) as usize;
 
         let mut parts = Vec::new();
-        let mut current_part: Vec<Range<Key>> = Vec::new();
+        let mut current_part = Vec::new();
         let mut current_part_size: usize = 0;
         for range in &self.ranges {
-            let last = current_part
-                .last()
-                .map(|r| r.end)
-                .unwrap_or(Key::from_i128(0));
-            let cut_here = (range.start >= METADATA_CUT && last < METADATA_CUT)
-                || (range.start >= BASEBACKUP_CUT && last < BASEBACKUP_CUT);
-
             // If appending the next contiguous range in the keyspace to the current
             // partition would cause it to be too large, start a new partition.
             let this_size = key_range_size(range) as usize;
-            if cut_here
-                || current_part_size + this_size > target_nblocks && !current_part.is_empty()
-            {
+            if current_part_size + this_size > target_nblocks && !current_part.is_empty() {
                 parts.push(KeySpace {
                     ranges: current_part,
                 });

pageserver/src/metrics.rs

@@ -40,6 +40,9 @@ pub enum StorageTimeOperation {
     #[strum(serialize = "logical size")]
     LogicalSize,
 
+    #[strum(serialize = "imitate logical size")]
+    ImitateLogicalSize,
+
     #[strum(serialize = "load layer map")]
     LoadLayerMap,
 
@@ -1361,6 +1364,7 @@ pub struct TimelineMetrics {
     pub compact_time_histo: StorageTimeMetrics,
     pub create_images_time_histo: StorageTimeMetrics,
     pub logical_size_histo: StorageTimeMetrics,
+    pub imitate_logical_size_histo: StorageTimeMetrics,
     pub load_layer_map_histo: StorageTimeMetrics,
     pub garbage_collect_histo: StorageTimeMetrics,
     pub last_record_gauge: IntGauge,
@@ -1389,6 +1393,11 @@ impl TimelineMetrics {
             StorageTimeMetrics::new(StorageTimeOperation::CreateImages, &tenant_id, &timeline_id);
         let logical_size_histo =
             StorageTimeMetrics::new(StorageTimeOperation::LogicalSize, &tenant_id, &timeline_id);
+        let imitate_logical_size_histo = StorageTimeMetrics::new(
+            StorageTimeOperation::ImitateLogicalSize,
+            &tenant_id,
+            &timeline_id,
+        );
         let load_layer_map_histo =
             StorageTimeMetrics::new(StorageTimeOperation::LoadLayerMap, &tenant_id, &timeline_id);
         let garbage_collect_histo =
@@ -1421,6 +1430,7 @@ impl TimelineMetrics {
             compact_time_histo,
             create_images_time_histo,
             logical_size_histo,
+            imitate_logical_size_histo,
             garbage_collect_histo,
             load_layer_map_histo,
             last_record_gauge,

pageserver/src/pgdatadir_mapping.rs

@@ -662,21 +662,20 @@ impl Timeline {
         ctx: &RequestContext,
     ) -> Result<KeySpace, CollectKeySpaceError> {
         // Iterate through key ranges, greedily packing them into partitions
-        // This function is responsible for appending keys in order, using implicit
-        // knowledge of how keys are defined.
         let mut result = KeySpaceAccum::new();
 
+        // The dbdir metadata always exists
+        result.add_key(DBDIR_KEY);
+
         // Fetch list of database dirs and iterate them
         let buf = self.get(DBDIR_KEY, lsn, ctx).await?;
         let dbdir = DbDirectory::des(&buf)?;
 
-        let mut metadata_keys = Vec::new();
-
         let mut dbs: Vec<(Oid, Oid)> = dbdir.dbdirs.keys().cloned().collect();
         dbs.sort_unstable();
         for (spcnode, dbnode) in dbs {
-            metadata_keys.push(relmap_file_key(spcnode, dbnode));
-            metadata_keys.push(rel_dir_to_key(spcnode, dbnode));
+            result.add_key(relmap_file_key(spcnode, dbnode));
+            result.add_key(rel_dir_to_key(spcnode, dbnode));
 
             let mut rels: Vec<RelTag> = self
                 .list_rels(spcnode, dbnode, lsn, ctx)
@@ -690,7 +689,7 @@ impl Timeline {
                 let relsize = buf.get_u32_le();
 
                 result.add_range(rel_block_to_key(rel, 0)..rel_block_to_key(rel, relsize));
-                metadata_keys.push(relsize_key);
+                result.add_key(relsize_key);
             }
         }
 
@@ -733,13 +732,6 @@ impl Timeline {
         if self.get(AUX_FILES_KEY, lsn, ctx).await.is_ok() {
             result.add_key(AUX_FILES_KEY);
         }
-
-        // The dbdir metadata always exists
-        result.add_key(DBDIR_KEY);
-        for key in metadata_keys {
-            result.add_key(key);
-        }
-
         Ok(result.to_keyspace())
     }
 
@@ -1482,11 +1474,21 @@ static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; BLCKSZ as usize]);
 //
 // Below is a full list of the keyspace allocation:
 //
-
+// DbDir:
+// 00 00000000 00000000 00000000 00   00000000
+//
+// Filenodemap:
+// 00 SPCNODE  DBNODE   00000000 00   00000000
+//
+// RelDir:
+// 00 SPCNODE  DBNODE   00000000 00   00000001 (Postgres never uses relfilenode 0)
 //
 // RelBlock:
 // 00 SPCNODE  DBNODE   RELNODE  FORK BLKNUM
 //
+// RelSize:
+// 00 SPCNODE  DBNODE   RELNODE  FORK FFFFFFFF
+//
 // SlruDir:
 // 01 kind     00000000 00000000 00   00000000
 //
@@ -1511,31 +1513,11 @@ static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; BLCKSZ as usize]);
 // AuxFiles:
 // 03 00000000 00000000 00000000 00   00000002
 //
-// DbDir:
-// 04 00000000 00000000 00000000 00   00000000
-//
-// Filenodemap:
-// 04 SPCNODE  DBNODE   00000000 00   00000000
-//
-// RelDir:
-// 04 SPCNODE  DBNODE   00000000 00   00000001 (Postgres never uses relfilenode 0)
-//
-// RelSize:
-// 04 SPCNODE  DBNODE   RELNODE  FORK FFFFFFFF
 
 //-- Section 01: relation data and metadata
 
-/// Keys above this Key are required to serve a basebackup request
-pub(crate) const BASEBACKUP_CUT: Key = slru_dir_to_key(SlruKind::Clog);
-
-/// Keys aboe this Key are needed to make a logical size calculation
-///
-/// Ensuring that such keys are stored above the main range of user relation
-/// blocks enables much more efficient space management.
-pub(crate) const METADATA_CUT: Key = CONTROLFILE_KEY;
-
 const DBDIR_KEY: Key = Key {
-    field1: 0x04,
+    field1: 0x00,
     field2: 0,
     field3: 0,
     field4: 0,
@@ -1545,14 +1527,14 @@ const DBDIR_KEY: Key = Key {
 
 fn dbdir_key_range(spcnode: Oid, dbnode: Oid) -> Range<Key> {
     Key {
-        field1: 0x04,
+        field1: 0x00,
         field2: spcnode,
         field3: dbnode,
         field4: 0,
         field5: 0,
         field6: 0,
     }..Key {
-        field1: 0x04,
+        field1: 0x00,
         field2: spcnode,
         field3: dbnode,
         field4: 0xffffffff,
@@ -1563,7 +1545,7 @@ fn dbdir_key_range(spcnode: Oid, dbnode: Oid) -> Range<Key> {
 
 fn relmap_file_key(spcnode: Oid, dbnode: Oid) -> Key {
     Key {
-        field1: 0x04,
+        field1: 0x00,
         field2: spcnode,
         field3: dbnode,
         field4: 0,
@@ -1574,7 +1556,7 @@ fn relmap_file_key(spcnode: Oid, dbnode: Oid) -> Key {
 
 fn rel_dir_to_key(spcnode: Oid, dbnode: Oid) -> Key {
     Key {
-        field1: 0x04,
+        field1: 0x00,
         field2: spcnode,
         field3: dbnode,
         field4: 0,
@@ -1596,7 +1578,7 @@ fn rel_block_to_key(rel: RelTag, blknum: BlockNumber) -> Key {
 
 fn rel_size_to_key(rel: RelTag) -> Key {
     Key {
-        field1: 0x04,
+        field1: 0x00,
         field2: rel.spcnode,
         field3: rel.dbnode,
         field4: rel.relnode,
@@ -1625,7 +1607,7 @@ fn rel_key_range(rel: RelTag) -> Range<Key> {
 
 //-- Section 02: SLRUs
 
-const fn slru_dir_to_key(kind: SlruKind) -> Key {
+fn slru_dir_to_key(kind: SlruKind) -> Key {
     Key {
         field1: 0x01,
         field2: match kind {

pageserver/src/tenant.rs

@@ -61,6 +61,7 @@ use self::mgr::TenantsMap;
 use self::remote_timeline_client::RemoteTimelineClient;
 use self::timeline::uninit::TimelineUninitMark;
 use self::timeline::uninit::UninitializedTimeline;
+use self::timeline::EvictionTaskTenantState;
 use self::timeline::TimelineResources;
 use crate::config::PageServerConf;
 use crate::context::{DownloadBehavior, RequestContext};
@@ -251,6 +252,8 @@ pub struct Tenant {
     cached_logical_sizes: tokio::sync::Mutex<HashMap<(TimelineId, Lsn), u64>>,
     cached_synthetic_tenant_size: Arc<AtomicU64>,
 
+    eviction_task_tenant_state: tokio::sync::Mutex<EvictionTaskTenantState>,
+
     pub(crate) delete_progress: Arc<tokio::sync::Mutex<DeleteTenantFlow>>,
 
     // Cancellation token fires when we have entered shutdown().  This is a parent of
@@ -2364,6 +2367,7 @@ impl Tenant {
             state,
             cached_logical_sizes: tokio::sync::Mutex::new(HashMap::new()),
             cached_synthetic_tenant_size: Arc::new(AtomicU64::new(0)),
+            eviction_task_tenant_state: tokio::sync::Mutex::new(EvictionTaskTenantState::default()),
             delete_progress: Arc::new(tokio::sync::Mutex::new(DeleteTenantFlow::default())),
             cancel: CancellationToken::default(),
             gate: Gate::new(format!("Tenant<{tenant_id}>")),

pageserver/src/tenant/storage_layer/layer_desc.rs

@@ -5,7 +5,7 @@ use utils::{
     lsn::Lsn,
 };
 
-use crate::{pgdatadir_mapping::METADATA_CUT, repository::Key};
+use crate::repository::Key;
 
 use super::{DeltaFileName, ImageFileName, LayerFileName};
 
@@ -49,20 +49,6 @@ impl PersistentLayerDesc {
         }
     }
 
-    /// Does this layer consist exclusively of metadata
-    /// content such as dbdir & relation sizes?  This is a
-    /// hint that the layer is likely to be small and should
-    /// not be a candidate for eviction under normal circumstances.
-    pub fn is_metadata_pages(&self) -> bool {
-        self.key_range.start >= METADATA_CUT
-    }
-
-    /// Does this layer consist exclusively of content
-    /// required to serve a basebackup request?
-    pub fn is_basebackup_pages(&self) -> bool {
-        self.key_range.start >= METADATA_CUT
-    }
-
     pub fn short_id(&self) -> impl Display {
         self.filename()
     }

pageserver/src/tenant/timeline.rs

@@ -81,6 +81,8 @@ use crate::task_mgr::TaskKind;
 use crate::ZERO_PAGE;
 
 use self::delete::DeleteTimelineFlow;
+pub(super) use self::eviction_task::EvictionTaskTenantState;
+use self::eviction_task::EvictionTaskTimelineState;
 use self::layer_manager::LayerManager;
 use self::logical_size::LogicalSize;
 use self::walreceiver::{WalReceiver, WalReceiverConf};
@@ -296,6 +298,8 @@ pub struct Timeline {
     /// timeline is being deleted. If 'true', the timeline has already been deleted.
     pub delete_progress: Arc<tokio::sync::Mutex<DeleteTimelineFlow>>,
 
+    eviction_task_timeline_state: tokio::sync::Mutex<EvictionTaskTimelineState>,
+
     /// Barrier to wait before doing initial logical size calculation. Used only during startup.
     initial_logical_size_can_start: Option<completion::Barrier>,
 
@@ -429,6 +433,7 @@ impl std::fmt::Display for PageReconstructError {
 pub enum LogicalSizeCalculationCause {
     Initial,
     ConsumptionMetricsSyntheticSize,
+    EvictionTaskImitation,
     TenantSizeHandler,
 }
 
@@ -1437,6 +1442,9 @@ impl Timeline {
 
                 state,
 
+                eviction_task_timeline_state: tokio::sync::Mutex::new(
+                    EvictionTaskTimelineState::default(),
+                ),
                 delete_progress: Arc::new(tokio::sync::Mutex::new(DeleteTimelineFlow::default())),
 
                 initial_logical_size_can_start,
@@ -1959,6 +1967,9 @@ impl Timeline {
             LogicalSizeCalculationCause::Initial
             | LogicalSizeCalculationCause::ConsumptionMetricsSyntheticSize
             | LogicalSizeCalculationCause::TenantSizeHandler => &self.metrics.logical_size_histo,
+            LogicalSizeCalculationCause::EvictionTaskImitation => {
+                &self.metrics.imitate_logical_size_histo
+            }
         };
         let timer = storage_time_metrics.start_timer();
         let logical_size = self
@@ -2735,18 +2746,18 @@ impl Timeline {
         partition_size: u64,
         ctx: &RequestContext,
     ) -> anyhow::Result<(KeyPartitioning, Lsn)> {
-        // {
-        //     let partitioning_guard = self.partitioning.lock().unwrap();
-        //     let distance = lsn.0 - partitioning_guard.1 .0;
-        //     if partitioning_guard.1 != Lsn(0) && distance <= self.repartition_threshold {
-        //         debug!(
-        //             distance,
-        //             threshold = self.repartition_threshold,
-        //             "no repartitioning needed"
-        //         );
-        //         return Ok((partitioning_guard.0.clone(), partitioning_guard.1));
-        //     }
-        // }
+        {
+            let partitioning_guard = self.partitioning.lock().unwrap();
+            let distance = lsn.0 - partitioning_guard.1 .0;
+            if partitioning_guard.1 != Lsn(0) && distance <= self.repartition_threshold {
+                debug!(
+                    distance,
+                    threshold = self.repartition_threshold,
+                    "no repartitioning needed"
+                );
+                return Ok((partitioning_guard.0.clone(), partitioning_guard.1));
+            }
+        }
         let keyspace = self.collect_keyspace(lsn, ctx).await?;
         let partitioning = keyspace.partition(partition_size);
 
@@ -4274,11 +4285,6 @@ impl Timeline {
             let file_size = l.file_size();
             max_layer_size = max_layer_size.map_or(Some(file_size), |m| Some(m.max(file_size)));
 
-            // Don't evict small layers required to serve a basebackup
-            if l.is_basebackup_pages() {
-                continue;
-            }
-
             let l = guard.get_from_desc(&l);
 
             let l = match l.keep_resident().await {

pageserver/src/tenant/timeline/eviction_task.rs

@@ -14,6 +14,7 @@
 //!
 //! See write-up on restart on-demand download spike: <https://gist.github.com/problame/2265bf7b8dc398be834abfead36c76b5>
 use std::{
+    collections::HashMap,
     ops::ControlFlow,
     sync::Arc,
     time::{Duration, SystemTime},
@@ -21,15 +22,17 @@ use std::{
 
 use tokio::time::Instant;
 use tokio_util::sync::CancellationToken;
-use tracing::{debug, error, info, instrument, warn};
+use tracing::{debug, error, info, info_span, instrument, warn, Instrument};
 
 use crate::{
     context::{DownloadBehavior, RequestContext},
+    pgdatadir_mapping::CollectKeySpaceError,
     task_mgr::{self, TaskKind, BACKGROUND_RUNTIME},
     tenant::{
         config::{EvictionPolicy, EvictionPolicyLayerAccessThreshold},
         tasks::{BackgroundLoopKind, RateLimitError},
         timeline::EvictionError,
+        LogicalSizeCalculationCause, Tenant,
     },
 };
 
@@ -37,6 +40,16 @@ use utils::completion;
 
 use super::Timeline;
 
+#[derive(Default)]
+pub struct EvictionTaskTimelineState {
+    last_layer_access_imitation: Option<tokio::time::Instant>,
+}
+
+#[derive(Default)]
+pub struct EvictionTaskTenantState {
+    last_layer_access_imitation: Option<Instant>,
+}
+
 impl Timeline {
     pub(super) fn launch_eviction_task(
         self: &Arc<Self>,
@@ -165,6 +178,7 @@ impl Timeline {
         //    that were accessed to compute the value in the first place.
         // 3. Invalidate the caches at a period of < p.threshold/2, so that the values
         //    get re-computed from layers, thereby counting towards layer access stats.
+        // 4. Make the eviction task imitate the layer accesses that typically hit caches.
         //
         // We follow approach (4) here because in Neon prod deployment:
         // - page cache is quite small => high churn => low hit rate
@@ -176,6 +190,10 @@ impl Timeline {
         //
         // We should probably move to persistent caches in the future, or avoid
         // having inactive tenants attached to pageserver in the first place.
+        match self.imitate_layer_accesses(p, cancel, ctx).await {
+            ControlFlow::Break(()) => return ControlFlow::Break(()),
+            ControlFlow::Continue(()) => (),
+        }
 
         #[allow(dead_code)]
         #[derive(Debug, Default)]
@@ -197,11 +215,6 @@ impl Timeline {
             let layers = guard.layer_map();
             let mut candidates = Vec::new();
             for hist_layer in layers.iter_historic_layers() {
-                // Don't evict the small layers needed to serve a basebackup request.
-                if hist_layer.is_basebackup_pages() {
-                    continue;
-                }
-
                 let hist_layer = guard.get_from_desc(&hist_layer);
 
                 // guard against eviction while we inspect it; it might be that eviction_task and
@@ -297,4 +310,170 @@ impl Timeline {
         }
         ControlFlow::Continue(())
     }
+
+    #[instrument(skip_all)]
+    async fn imitate_layer_accesses(
+        &self,
+        p: &EvictionPolicyLayerAccessThreshold,
+        cancel: &CancellationToken,
+        ctx: &RequestContext,
+    ) -> ControlFlow<()> {
+        let mut state = self.eviction_task_timeline_state.lock().await;
+
+        // Only do the imitate_layer accesses approximately as often as the threshold.  A little
+        // more frequently, to avoid this period racing with the threshold/period-th eviction iteration.
+        let inter_imitate_period = p.threshold.checked_sub(p.period).unwrap_or(p.threshold);
+
+        match state.last_layer_access_imitation {
+            Some(ts) if ts.elapsed() < inter_imitate_period => { /* no need to run */ }
+            _ => {
+                self.imitate_timeline_cached_layer_accesses(ctx).await;
+                state.last_layer_access_imitation = Some(tokio::time::Instant::now())
+            }
+        }
+        drop(state);
+
+        if cancel.is_cancelled() {
+            return ControlFlow::Break(());
+        }
+
+        // This task is timeline-scoped, but the synthetic size calculation is tenant-scoped.
+        // Make one of the tenant's timelines draw the short straw and run the calculation.
+        // The others wait until the calculation is done so that they take into account the
+        // imitated accesses that the winner made.
+        let tenant = match crate::tenant::mgr::get_tenant(self.tenant_id, true) {
+            Ok(t) => t,
+            Err(_) => {
+                return ControlFlow::Break(());
+            }
+        };
+        let mut state = tenant.eviction_task_tenant_state.lock().await;
+        match state.last_layer_access_imitation {
+            Some(ts) if ts.elapsed() < inter_imitate_period => { /* no need to run */ }
+            _ => {
+                self.imitate_synthetic_size_calculation_worker(&tenant, ctx, cancel)
+                    .await;
+                state.last_layer_access_imitation = Some(tokio::time::Instant::now());
+            }
+        }
+        drop(state);
+
+        if cancel.is_cancelled() {
+            return ControlFlow::Break(());
+        }
+
+        ControlFlow::Continue(())
+    }
+
+    /// Recompute the values which would cause on-demand downloads during restart.
+    #[instrument(skip_all)]
+    async fn imitate_timeline_cached_layer_accesses(&self, ctx: &RequestContext) {
+        let lsn = self.get_last_record_lsn();
+
+        // imitiate on-restart initial logical size
+        let size = self
+            .calculate_logical_size(lsn, LogicalSizeCalculationCause::EvictionTaskImitation, ctx)
+            .instrument(info_span!("calculate_logical_size"))
+            .await;
+
+        match &size {
+            Ok(_size) => {
+                // good, don't log it to avoid confusion
+            }
+            Err(_) => {
+                // we have known issues for which we already log this on consumption metrics,
+                // gc, and compaction. leave logging out for now.
+                //
+                // https://github.com/neondatabase/neon/issues/2539
+            }
+        }
+
+        // imitiate repartiting on first compactation
+        if let Err(e) = self
+            .collect_keyspace(lsn, ctx)
+            .instrument(info_span!("collect_keyspace"))
+            .await
+        {
+            // if this failed, we probably failed logical size because these use the same keys
+            if size.is_err() {
+                // ignore, see above comment
+            } else {
+                match e {
+                    CollectKeySpaceError::Cancelled => {
+                        // Shutting down, ignore
+                    }
+                    err => {
+                        warn!(
+                            "failed to collect keyspace but succeeded in calculating logical size: {err:#}"
+                        );
+                    }
+                }
+            }
+        }
+    }
+
+    // Imitate the synthetic size calculation done by the consumption_metrics module.
+    #[instrument(skip_all)]
+    async fn imitate_synthetic_size_calculation_worker(
+        &self,
+        tenant: &Arc<Tenant>,
+        ctx: &RequestContext,
+        cancel: &CancellationToken,
+    ) {
+        if self.conf.metric_collection_endpoint.is_none() {
+            // We don't start the consumption metrics task if this is not set in the config.
+            // So, no need to imitate the accesses in that case.
+            return;
+        }
+
+        // The consumption metrics are collected on a per-tenant basis, by a single
+        // global background loop.
+        // It limits the number of synthetic size calculations using the global
+        // `concurrent_tenant_size_logical_size_queries` semaphore to not overload
+        // the pageserver. (size calculation is somewhat expensive in terms of CPU and IOs).
+        //
+        // If we used that same semaphore here, then we'd compete for the
+        // same permits, which may impact timeliness of consumption metrics.
+        // That is a no-go, as consumption metrics are much more important
+        // than what we do here.
+        //
+        // So, we have a separate semaphore, initialized to the same
+        // number of permits as the `concurrent_tenant_size_logical_size_queries`.
+        // In the worst, we would have twice the amount of concurrenct size calculations.
+        // But in practice, the `p.threshold` >> `consumption metric interval`, and
+        // we spread out the eviction task using `random_init_delay`.
+        // So, the chance of the worst case is quite low in practice.
+        // It runs as a per-tenant task, but the eviction_task.rs is per-timeline.
+        // So, we must coordinate with other with other eviction tasks of this tenant.
+        let limit = self
+            .conf
+            .eviction_task_immitated_concurrent_logical_size_queries
+            .inner();
+
+        let mut throwaway_cache = HashMap::new();
+        let gather = crate::tenant::size::gather_inputs(
+            tenant,
+            limit,
+            None,
+            &mut throwaway_cache,
+            LogicalSizeCalculationCause::EvictionTaskImitation,
+            ctx,
+        )
+        .instrument(info_span!("gather_inputs"));
+
+        tokio::select! {
+            _ = cancel.cancelled() => {}
+            gather_result = gather => {
+                match gather_result {
+                    Ok(_) => {},
+                    Err(e) => {
+                        // We don't care about the result, but, if it failed, we should log it,
+                        // since consumption metric might be hitting the cached value and
+                        // thus not encountering this error.
+                        warn!("failed to imitate synthetic size calculation accesses: {e:#}")
+                    }
+                }
+           }
+        }
+    }
 }

proxy/src/auth/credentials.rs

@@ -1,7 +1,9 @@
 //! User credentials used in authentication.
 
 use crate::{
-    auth::password_hack::parse_endpoint_param, error::UserFacingError, proxy::neon_options,
+    auth::password_hack::parse_endpoint_param,
+    error::UserFacingError,
+    proxy::{neon_options, NUM_CONNECTION_ACCEPTED_BY_SNI},
 };
 use itertools::Itertools;
 use pq_proto::StartupMessageParams;
@@ -124,6 +126,22 @@ impl<'a> ClientCredentials<'a> {
         .transpose()?;
 
         info!(user, project = project.as_deref(), "credentials");
+        if sni.is_some() {
+            info!("Connection with sni");
+            NUM_CONNECTION_ACCEPTED_BY_SNI
+                .with_label_values(&["sni"])
+                .inc();
+        } else if project.is_some() {
+            NUM_CONNECTION_ACCEPTED_BY_SNI
+                .with_label_values(&["no_sni"])
+                .inc();
+            info!("Connection without sni");
+        } else {
+            NUM_CONNECTION_ACCEPTED_BY_SNI
+                .with_label_values(&["password_hack"])
+                .inc();
+            info!("Connection with password hack");
+        }
 
         let cache_key = format!(
             "{}{}",

proxy/src/proxy.rs

@@ -129,6 +129,15 @@ pub static RATE_LIMITER_LIMIT: Lazy<IntGaugeVec> = Lazy::new(|| {
     .unwrap()
 });
 
+pub static NUM_CONNECTION_ACCEPTED_BY_SNI: Lazy<IntCounterVec> = Lazy::new(|| {
+    register_int_counter_vec!(
+        "proxy_accepted_connections_by_sni",
+        "Number of connections (per sni).",
+        &["kind"],
+    )
+    .unwrap()
+});
+
 pub struct LatencyTimer {
     // time since the stopwatch was started
     start: Option<Instant>,

proxy/src/serverless/conn_pool.rs

@@ -210,12 +210,7 @@ impl GlobalConnPool {
                 client.session.send(session_id)?;
                 latency_timer.pool_hit();
                 latency_timer.success();
-                return Ok(Client {
-                    conn_id: client.conn_id,
-                    inner: Some(client),
-                    span: Span::current(),
-                    pool,
-                });
+                return Ok(Client::new(client, pool).await);
             }
         } else {
             let conn_id = uuid::Uuid::new_v4();
@@ -263,15 +258,11 @@ impl GlobalConnPool {
             _ => {}
         }
 
-        new_client.map(|inner| Client {
-            conn_id: inner.conn_id,
-            inner: Some(inner),
-            span: Span::current(),
-            pool,
-        })
+        // new_client.map(|inner| Client::new(inner, pool).await)
+        Ok(Client::new(new_client?, pool).await)
     }
 
-    fn put(&self, conn_info: &ConnInfo, client: ClientInner) -> anyhow::Result<()> {
+    fn put(&self, conn_info: &ConnInfo, client: ClientInner, pid: i32) -> anyhow::Result<()> {
         let conn_id = client.conn_id;
 
         // We want to hold this open while we return. This ensures that the pool can't close
@@ -315,9 +306,9 @@ impl GlobalConnPool {
 
         // do logging outside of the mutex
         if returned {
-            info!(%conn_id, "pool: returning connection '{conn_info}' back to the pool, total_conns={total_conns}, for this (db, user)={per_db_size}");
+            info!(%conn_id, "pool: returning connection '{conn_info}' back to the pool, total_conns={total_conns}, for this (db, user)={per_db_size}, pid={pid}");
         } else {
-            info!(%conn_id, "pool: throwing away connection '{conn_info}' because pool is full, total_conns={total_conns}");
+            info!(%conn_id, "pool: throwing away connection '{conn_info}' because pool is full, total_conns={total_conns}, pid={pid}");
         }
 
         Ok(())
@@ -528,6 +519,22 @@ struct ClientInner {
     conn_id: uuid::Uuid,
 }
 
+impl ClientInner {
+    pub async fn get_pid(&mut self) -> anyhow::Result<i32> {
+        let rows = self.inner.query("select pg_backend_pid();", &[]).await?;
+        if rows.len() != 1 {
+            Err(anyhow::anyhow!(
+                "expected 1 row from pg_backend_pid(), got {}",
+                rows.len()
+            ))
+        } else {
+            let pid = rows[0].get(0);
+            info!(%pid, "got pid");
+            Ok(pid)
+        }
+    }
+}
+
 impl Client {
     pub fn metrics(&self) -> Arc<MetricCounter> {
         USAGE_METRICS.register(self.inner.as_ref().unwrap().ids.clone())
@@ -539,6 +546,7 @@ pub struct Client {
     span: Span,
     inner: Option<ClientInner>,
     pool: Option<(ConnInfo, Arc<GlobalConnPool>)>,
+    pid: i32,
 }
 
 pub struct Discard<'a> {
@@ -547,12 +555,25 @@ pub struct Discard<'a> {
 }
 
 impl Client {
+    pub(self) async fn new(
+        mut inner: ClientInner,
+        pool: Option<(ConnInfo, Arc<GlobalConnPool>)>,
+    ) -> Self {
+        Self {
+            conn_id: inner.conn_id,
+            pid: inner.get_pid().await.unwrap_or(-1),
+            inner: Some(inner),
+            span: Span::current(),
+            pool,
+        }
+    }
     pub fn inner(&mut self) -> (&mut tokio_postgres::Client, Discard<'_>) {
         let Self {
             inner,
             pool,
             conn_id,
             span: _,
+            pid: _,
         } = self;
         (
             &mut inner
@@ -609,10 +630,11 @@ impl Drop for Client {
             .expect("client inner should not be removed");
         if let Some((conn_info, conn_pool)) = self.pool.take() {
             let current_span = self.span.clone();
+            let pid = self.pid;
             // return connection to the pool
             tokio::task::spawn_blocking(move || {
                 let _span = current_span.enter();
-                let _ = conn_pool.put(&conn_info, client);
+                let _ = conn_pool.put(&conn_info, client, pid);
             });
         }
     }

vm-image-spec.yaml

@@ -0,0 +1,143 @@
+# Supplemental file for neondatabase/autoscaling's vm-builder, for producing the VM compute image.
+---
+commands:
+  - name: cgconfigparser
+    user: root
+    sysvInitAction: sysinit
+    shell: "cgconfigparser -l /etc/cgconfig.conf -s 1664"
+  - name: pgbouncer
+    user: nobody
+    sysvInitAction: respawn
+    shell: "/usr/local/bin/pgbouncer /etc/pgbouncer.ini"
+  - name: postgres-exporter
+    user: nobody
+    sysvInitAction: respawn
+    shell: 'DATA_SOURCE_NAME="user=cloud_admin sslmode=disable dbname=postgres" /bin/postgres_exporter --extend.query-path /etc/postgres_exporter_queries.yml'
+shutdownHook: |
+  su -p postgres --session-command '/usr/local/bin/pg_ctl stop -D /var/db/postgres/compute/pgdata -m fast --wait -t 10'
+files:
+  - filename: pgbouncer.ini
+    content: |
+      [databases]
+      *=host=localhost port=5432 auth_user=cloud_admin
+      [pgbouncer]
+      listen_port=6432
+      listen_addr=0.0.0.0
+      auth_type=scram-sha-256
+      auth_user=cloud_admin
+      auth_dbname=postgres
+      client_tls_sslmode=disable
+      server_tls_sslmode=disable
+      pool_mode=transaction
+      max_client_conn=10000
+      default_pool_size=16
+      max_prepared_statements=0
+  - filename: cgconfig.conf
+    content: |
+      # Configuration for cgroups in VM compute nodes
+      group neon-postgres {
+          perm {
+              admin {
+                  uid = postgres;
+              }
+              task {
+                  gid = users;
+              }
+          }
+          memory {}
+      }
+  - filename: postgres_exporter_queries.yml
+    content: |
+      postgres_exporter_pg_database_size:
+        query: "SELECT pg_database.datname, pg_database_size(pg_database.datname) as bytes, 42 as fourtytwo FROM pg_database"
+        cache_seconds: 30
+        metrics:
+          - datname:
+              usage: "LABEL"
+              description: "Name of the database"
+          - bytes:
+              usage: "GAUGE"
+              description: "Disk space used by the database"
+          - fourtytwo:
+              usage: "GAUGE"
+              description: "fourtytwo"
+build: |
+  # Build cgroup-tools
+  #
+  # At time of writing (2023-03-14), debian bullseye has a version of cgroup-tools (technically
+  # libcgroup) that doesn't support cgroup v2 (version 0.41-11). Unfortunately, the vm-monitor
+  # requires cgroup v2, so we'll build cgroup-tools ourselves.
+  FROM debian:bullseye-slim as libcgroup-builder
+  ENV LIBCGROUP_VERSION v2.0.3
+
+  RUN set -exu \
+      && apt update \
+      && apt install --no-install-recommends -y \
+          git \
+          ca-certificates \
+          automake \
+          cmake \
+          make \
+          gcc \
+          byacc \
+          flex \
+          libtool \
+          libpam0g-dev \
+      && git clone --depth 1 -b $LIBCGROUP_VERSION https://github.com/libcgroup/libcgroup \
+      && INSTALL_DIR="/libcgroup-install" \
+      && mkdir -p "$INSTALL_DIR/bin" "$INSTALL_DIR/include" \
+      && cd libcgroup \
+      # extracted from bootstrap.sh, with modified flags:
+      && (test -d m4 || mkdir m4) \
+      && autoreconf -fi \
+      && rm -rf autom4te.cache \
+      && CFLAGS="-O3" ./configure --prefix="$INSTALL_DIR" --sysconfdir=/etc --localstatedir=/var --enable-opaque-hierarchy="name=systemd" \
+      # actually build the thing...
+      && make install
+
+  FROM quay.io/prometheuscommunity/postgres-exporter:v0.12.0 AS postgres-exporter
+
+  # Build pgbouncer
+  #
+  FROM debian:bullseye-slim AS pgbouncer
+  RUN set -e \
+      && apt-get update \
+      && apt-get install -y \
+          curl \
+          build-essential \
+          pkg-config \
+          libevent-dev \
+          libssl-dev
+
+  ENV PGBOUNCER_VERSION 1.21.0
+  ENV PGBOUNCER_GITPATH 1_21_0
+  RUN set -e \
+      && curl -sfSL https://github.com/pgbouncer/pgbouncer/releases/download/pgbouncer_${PGBOUNCER_GITPATH}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz -o pgbouncer-${PGBOUNCER_VERSION}.tar.gz \
+      && tar xzvf pgbouncer-${PGBOUNCER_VERSION}.tar.gz \
+      && cd pgbouncer-${PGBOUNCER_VERSION} \
+      && LDFLAGS=-static ./configure --prefix=/usr/local/pgbouncer --without-openssl \
+      && make -j $(nproc) \
+      && make install
+merge: |
+  # tweak nofile limits
+  RUN set -e \
+      && echo 'fs.file-max = 1048576' >>/etc/sysctl.conf \
+      && test ! -e /etc/security || ( \
+         echo '*    - nofile 1048576' >>/etc/security/limits.conf \
+      && echo 'root - nofile 1048576' >>/etc/security/limits.conf \
+         )
+
+  COPY cgconfig.conf /etc/cgconfig.conf
+  COPY pgbouncer.ini /etc/pgbouncer.ini
+  COPY postgres_exporter_queries.yml /etc/postgres_exporter_queries.yml
+  RUN set -e \
+      && chown postgres:postgres /etc/pgbouncer.ini \
+      && chmod 0644 /etc/pgbouncer.ini \
+      && chmod 0644 /etc/cgconfig.conf \
+      && chmod 0644 /etc/postgres_exporter_queries.yml
+
+  COPY --from=libcgroup-builder /libcgroup-install/bin/*  /usr/bin/
+  COPY --from=libcgroup-builder /libcgroup-install/lib/*  /usr/lib/
+  COPY --from=libcgroup-builder /libcgroup-install/sbin/* /usr/sbin/
+  COPY --from=postgres-exporter /bin/postgres_exporter /bin/postgres_exporter
+  COPY --from=pgbouncer         /usr/local/pgbouncer/bin/pgbouncer /usr/local/bin/pgbouncer