From 4904613aaa70c1f4b63419874cc4b5205c60c596 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arpad=20M=C3=BCller?= <arpad-m@users.noreply.github.com>
Date: Tue, 5 Sep 2023 12:55:45 +0200
Subject: [PATCH 01/61] Convert `VirtualFile::{seek,metadata}` to async (#5195)

## Problem

We want to convert the `VirtualFile` APIs to async fn so that we can
adopt one of the async I/O solutions.

## Summary of changes

Convert the following APIs of `VirtualFile` to async fn (as well as all
of the APIs calling it):

* `VirtualFile::seek`
* `VirtualFile::metadata`
* Also, prepare for deletion of the write impl by writing the summary to
a buffer before writing it to disk, as suggested in
https://github.com/neondatabase/neon/issues/4743#issuecomment-1700663864
. This change adds an additional warning for the case when the summary
exceeds a block. Previously, we'd have silently corrupted data in this
(unlikely) case.
* `WriteBlobWriter::write_blob`, in preparation for making
`VirtualFile::write_all` async.
---
 pageserver/src/tenant/blob_io.rs              | 15 ++---
 .../src/tenant/storage_layer/delta_layer.rs   | 57 +++++++++++--------
 .../src/tenant/storage_layer/image_layer.rs   | 50 +++++++++-------
 .../tenant/storage_layer/inmemory_layer.rs    |  9 ++-
 pageserver/src/tenant/timeline.rs             | 48 +++++++++-------
 pageserver/src/virtual_file.rs                | 26 ++++-----
 6 files changed, 114 insertions(+), 91 deletions(-)

diff --git a/pageserver/src/tenant/blob_io.rs b/pageserver/src/tenant/blob_io.rs
index e4dede2c30..ba3542d869 100644
--- a/pageserver/src/tenant/blob_io.rs
+++ b/pageserver/src/tenant/blob_io.rs
@@ -83,15 +83,6 @@ impl<'a> BlockCursor<'a> {
     }
 }
 
-///
-/// Abstract trait for a data sink that you can write blobs to.
-///
-pub trait BlobWriter {
-    /// Write a blob of data. Returns the offset that it was written to,
-    /// which can be used to retrieve the data later.
-    fn write_blob(&mut self, srcbuf: &[u8]) -> Result<u64, Error>;
-}
-
 ///
 /// An implementation of BlobWriter to write blobs to anything that
 /// implements std::io::Write.
@@ -123,11 +114,13 @@ impl<W> WriteBlobWriter<W> {
     }
 }
 
-impl<W> BlobWriter for WriteBlobWriter<W>
+impl<W> WriteBlobWriter<W>
 where
     W: std::io::Write,
 {
-    fn write_blob(&mut self, srcbuf: &[u8]) -> Result<u64, Error> {
+    /// Write a blob of data. Returns the offset that it was written to,
+    /// which can be used to retrieve the data later.
+    pub async fn write_blob(&mut self, srcbuf: &[u8]) -> Result<u64, Error> {
         let offset = self.offset;
 
         if srcbuf.len() < 128 {
diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs
index 60427a22e4..e95c422d29 100644
--- a/pageserver/src/tenant/storage_layer/delta_layer.rs
+++ b/pageserver/src/tenant/storage_layer/delta_layer.rs
@@ -31,7 +31,7 @@ use crate::config::PageServerConf;
 use crate::context::RequestContext;
 use crate::page_cache::PAGE_SZ;
 use crate::repository::{Key, Value, KEY_SIZE};
-use crate::tenant::blob_io::{BlobWriter, WriteBlobWriter};
+use crate::tenant::blob_io::WriteBlobWriter;
 use crate::tenant::block_io::{BlockBuf, BlockCursor, BlockLease, BlockReader, FileBlockReader};
 use crate::tenant::disk_btree::{DiskBtreeBuilder, DiskBtreeReader, VisitDirection};
 use crate::tenant::storage_layer::{
@@ -590,7 +590,7 @@ impl DeltaLayerWriterInner {
     ///
     /// Start building a new delta layer.
     ///
-    fn new(
+    async fn new(
         conf: &'static PageServerConf,
         timeline_id: TimelineId,
         tenant_id: TenantId,
@@ -607,7 +607,7 @@ impl DeltaLayerWriterInner {
 
         let mut file = VirtualFile::create(&path)?;
         // make room for the header block
-        file.seek(SeekFrom::Start(PAGE_SZ as u64))?;
+        file.seek(SeekFrom::Start(PAGE_SZ as u64)).await?;
         let buf_writer = BufWriter::new(file);
         let blob_writer = WriteBlobWriter::new(buf_writer, PAGE_SZ as u64);
 
@@ -632,11 +632,12 @@ impl DeltaLayerWriterInner {
     ///
     /// The values must be appended in key, lsn order.
     ///
-    fn put_value(&mut self, key: Key, lsn: Lsn, val: Value) -> anyhow::Result<()> {
+    async fn put_value(&mut self, key: Key, lsn: Lsn, val: Value) -> anyhow::Result<()> {
         self.put_value_bytes(key, lsn, &Value::ser(&val)?, val.will_init())
+            .await
     }
 
-    fn put_value_bytes(
+    async fn put_value_bytes(
         &mut self,
         key: Key,
         lsn: Lsn,
@@ -645,7 +646,7 @@ impl DeltaLayerWriterInner {
     ) -> anyhow::Result<()> {
         assert!(self.lsn_range.start <= lsn);
 
-        let off = self.blob_writer.write_blob(val)?;
+        let off = self.blob_writer.write_blob(val).await?;
 
         let blob_ref = BlobRef::new(off, will_init);
 
@@ -662,7 +663,7 @@ impl DeltaLayerWriterInner {
     ///
     /// Finish writing the delta layer.
     ///
-    fn finish(self, key_end: Key) -> anyhow::Result<DeltaLayer> {
+    async fn finish(self, key_end: Key) -> anyhow::Result<DeltaLayer> {
         let index_start_blk =
             ((self.blob_writer.size() + PAGE_SZ as u64 - 1) / PAGE_SZ as u64) as u32;
 
@@ -671,7 +672,8 @@ impl DeltaLayerWriterInner {
 
         // Write out the index
         let (index_root_blk, block_buf) = self.tree.finish()?;
-        file.seek(SeekFrom::Start(index_start_blk as u64 * PAGE_SZ as u64))?;
+        file.seek(SeekFrom::Start(index_start_blk as u64 * PAGE_SZ as u64))
+            .await?;
         for buf in block_buf.blocks {
             file.write_all(buf.as_ref())?;
         }
@@ -687,11 +689,22 @@ impl DeltaLayerWriterInner {
             index_start_blk,
             index_root_blk,
         };
-        file.seek(SeekFrom::Start(0))?;
-        Summary::ser_into(&summary, &mut file)?;
+
+        let mut buf = smallvec::SmallVec::<[u8; PAGE_SZ]>::new();
+        Summary::ser_into(&summary, &mut buf)?;
+        if buf.spilled() {
+            // This is bad as we only have one free block for the summary
+            warn!(
+                "Used more than one page size for summary buffer: {}",
+                buf.len()
+            );
+        }
+        file.seek(SeekFrom::Start(0)).await?;
+        file.write_all(&buf)?;
 
         let metadata = file
             .metadata()
+            .await
             .context("get file metadata to determine size")?;
 
         // 5GB limit for objects without multipart upload (which we don't want to use)
@@ -774,7 +787,7 @@ impl DeltaLayerWriter {
     ///
     /// Start building a new delta layer.
     ///
-    pub fn new(
+    pub async fn new(
         conf: &'static PageServerConf,
         timeline_id: TimelineId,
         tenant_id: TenantId,
@@ -782,13 +795,10 @@ impl DeltaLayerWriter {
         lsn_range: Range<Lsn>,
     ) -> anyhow::Result<Self> {
         Ok(Self {
-            inner: Some(DeltaLayerWriterInner::new(
-                conf,
-                timeline_id,
-                tenant_id,
-                key_start,
-                lsn_range,
-            )?),
+            inner: Some(
+                DeltaLayerWriterInner::new(conf, timeline_id, tenant_id, key_start, lsn_range)
+                    .await?,
+            ),
         })
     }
 
@@ -797,11 +807,11 @@ impl DeltaLayerWriter {
     ///
     /// The values must be appended in key, lsn order.
     ///
-    pub fn put_value(&mut self, key: Key, lsn: Lsn, val: Value) -> anyhow::Result<()> {
-        self.inner.as_mut().unwrap().put_value(key, lsn, val)
+    pub async fn put_value(&mut self, key: Key, lsn: Lsn, val: Value) -> anyhow::Result<()> {
+        self.inner.as_mut().unwrap().put_value(key, lsn, val).await
     }
 
-    pub fn put_value_bytes(
+    pub async fn put_value_bytes(
         &mut self,
         key: Key,
         lsn: Lsn,
@@ -812,6 +822,7 @@ impl DeltaLayerWriter {
             .as_mut()
             .unwrap()
             .put_value_bytes(key, lsn, val, will_init)
+            .await
     }
 
     pub fn size(&self) -> u64 {
@@ -821,8 +832,8 @@ impl DeltaLayerWriter {
     ///
     /// Finish writing the delta layer.
     ///
-    pub fn finish(mut self, key_end: Key) -> anyhow::Result<DeltaLayer> {
-        self.inner.take().unwrap().finish(key_end)
+    pub async fn finish(mut self, key_end: Key) -> anyhow::Result<DeltaLayer> {
+        self.inner.take().unwrap().finish(key_end).await
     }
 }
 
diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs
index f329041fb1..62bfeb8eec 100644
--- a/pageserver/src/tenant/storage_layer/image_layer.rs
+++ b/pageserver/src/tenant/storage_layer/image_layer.rs
@@ -27,7 +27,7 @@ use crate::config::PageServerConf;
 use crate::context::RequestContext;
 use crate::page_cache::PAGE_SZ;
 use crate::repository::{Key, KEY_SIZE};
-use crate::tenant::blob_io::{BlobWriter, WriteBlobWriter};
+use crate::tenant::blob_io::WriteBlobWriter;
 use crate::tenant::block_io::{BlockBuf, BlockReader, FileBlockReader};
 use crate::tenant::disk_btree::{DiskBtreeBuilder, DiskBtreeReader, VisitDirection};
 use crate::tenant::storage_layer::{
@@ -519,7 +519,7 @@ impl ImageLayerWriterInner {
     ///
     /// Start building a new image layer.
     ///
-    fn new(
+    async fn new(
         conf: &'static PageServerConf,
         timeline_id: TimelineId,
         tenant_id: TenantId,
@@ -543,7 +543,7 @@ impl ImageLayerWriterInner {
             std::fs::OpenOptions::new().write(true).create_new(true),
         )?;
         // make room for the header block
-        file.seek(SeekFrom::Start(PAGE_SZ as u64))?;
+        file.seek(SeekFrom::Start(PAGE_SZ as u64)).await?;
         let blob_writer = WriteBlobWriter::new(file, PAGE_SZ as u64);
 
         // Initialize the b-tree index builder
@@ -569,9 +569,9 @@ impl ImageLayerWriterInner {
     ///
     /// The page versions must be appended in blknum order.
     ///
-    fn put_image(&mut self, key: Key, img: &[u8]) -> anyhow::Result<()> {
+    async fn put_image(&mut self, key: Key, img: &[u8]) -> anyhow::Result<()> {
         ensure!(self.key_range.contains(&key));
-        let off = self.blob_writer.write_blob(img)?;
+        let off = self.blob_writer.write_blob(img).await?;
 
         let mut keybuf: [u8; KEY_SIZE] = [0u8; KEY_SIZE];
         key.write_to_byte_slice(&mut keybuf);
@@ -583,14 +583,15 @@ impl ImageLayerWriterInner {
     ///
     /// Finish writing the image layer.
     ///
-    fn finish(self) -> anyhow::Result<ImageLayer> {
+    async fn finish(self) -> anyhow::Result<ImageLayer> {
         let index_start_blk =
             ((self.blob_writer.size() + PAGE_SZ as u64 - 1) / PAGE_SZ as u64) as u32;
 
         let mut file = self.blob_writer.into_inner();
 
         // Write out the index
-        file.seek(SeekFrom::Start(index_start_blk as u64 * PAGE_SZ as u64))?;
+        file.seek(SeekFrom::Start(index_start_blk as u64 * PAGE_SZ as u64))
+            .await?;
         let (index_root_blk, block_buf) = self.tree.finish()?;
         for buf in block_buf.blocks {
             file.write_all(buf.as_ref())?;
@@ -607,11 +608,22 @@ impl ImageLayerWriterInner {
             index_start_blk,
             index_root_blk,
         };
-        file.seek(SeekFrom::Start(0))?;
-        Summary::ser_into(&summary, &mut file)?;
+
+        let mut buf = smallvec::SmallVec::<[u8; PAGE_SZ]>::new();
+        Summary::ser_into(&summary, &mut buf)?;
+        if buf.spilled() {
+            // This is bad as we only have one free block for the summary
+            warn!(
+                "Used more than one page size for summary buffer: {}",
+                buf.len()
+            );
+        }
+        file.seek(SeekFrom::Start(0)).await?;
+        file.write_all(&buf)?;
 
         let metadata = file
             .metadata()
+            .await
             .context("get metadata to determine file size")?;
 
         let desc = PersistentLayerDesc::new_img(
@@ -687,7 +699,7 @@ impl ImageLayerWriter {
     ///
     /// Start building a new image layer.
     ///
-    pub fn new(
+    pub async fn new(
         conf: &'static PageServerConf,
         timeline_id: TimelineId,
         tenant_id: TenantId,
@@ -695,13 +707,9 @@ impl ImageLayerWriter {
         lsn: Lsn,
     ) -> anyhow::Result<ImageLayerWriter> {
         Ok(Self {
-            inner: Some(ImageLayerWriterInner::new(
-                conf,
-                timeline_id,
-                tenant_id,
-                key_range,
-                lsn,
-            )?),
+            inner: Some(
+                ImageLayerWriterInner::new(conf, timeline_id, tenant_id, key_range, lsn).await?,
+            ),
         })
     }
 
@@ -710,15 +718,15 @@ impl ImageLayerWriter {
     ///
     /// The page versions must be appended in blknum order.
     ///
-    pub fn put_image(&mut self, key: Key, img: &[u8]) -> anyhow::Result<()> {
-        self.inner.as_mut().unwrap().put_image(key, img)
+    pub async fn put_image(&mut self, key: Key, img: &[u8]) -> anyhow::Result<()> {
+        self.inner.as_mut().unwrap().put_image(key, img).await
     }
 
     ///
     /// Finish writing the image layer.
     ///
-    pub fn finish(mut self) -> anyhow::Result<ImageLayer> {
-        self.inner.take().unwrap().finish()
+    pub async fn finish(mut self) -> anyhow::Result<ImageLayer> {
+        self.inner.take().unwrap().finish().await
     }
 }
 
diff --git a/pageserver/src/tenant/storage_layer/inmemory_layer.rs b/pageserver/src/tenant/storage_layer/inmemory_layer.rs
index 35a77a7331..374b0bb60c 100644
--- a/pageserver/src/tenant/storage_layer/inmemory_layer.rs
+++ b/pageserver/src/tenant/storage_layer/inmemory_layer.rs
@@ -333,7 +333,8 @@ impl InMemoryLayer {
             self.tenant_id,
             Key::MIN,
             self.start_lsn..end_lsn,
-        )?;
+        )
+        .await?;
 
         let mut buf = Vec::new();
 
@@ -348,11 +349,13 @@ impl InMemoryLayer {
             for (lsn, pos) in vec_map.as_slice() {
                 cursor.read_blob_into_buf(*pos, &mut buf).await?;
                 let will_init = Value::des(&buf)?.will_init();
-                delta_layer_writer.put_value_bytes(key, *lsn, &buf, will_init)?;
+                delta_layer_writer
+                    .put_value_bytes(key, *lsn, &buf, will_init)
+                    .await?;
             }
         }
 
-        let delta_layer = delta_layer_writer.finish(Key::MAX)?;
+        let delta_layer = delta_layer_writer.finish(Key::MAX).await?;
         Ok(delta_layer)
     }
 }
diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs
index 816af214a5..d7ff70a29c 100644
--- a/pageserver/src/tenant/timeline.rs
+++ b/pageserver/src/tenant/timeline.rs
@@ -3033,7 +3033,8 @@ impl Timeline {
                     self.tenant_id,
                     &img_range,
                     lsn,
-                )?;
+                )
+                .await?;
 
                 fail_point!("image-layer-writer-fail-before-finish", |_| {
                     Err(PageReconstructError::Other(anyhow::anyhow!(
@@ -3069,11 +3070,11 @@ impl Timeline {
                                 }
                             }
                         };
-                        image_layer_writer.put_image(key, &img)?;
+                        image_layer_writer.put_image(key, &img).await?;
                         key = key.next();
                     }
                 }
-                let image_layer = image_layer_writer.finish()?;
+                let image_layer = image_layer_writer.finish().await?;
                 image_layers.push(image_layer);
             }
         }
@@ -3618,7 +3619,11 @@ impl Timeline {
                     {
                         // ... if so, flush previous layer and prepare to write new one
                         new_layers.push(Arc::new(
-                            writer.take().unwrap().finish(prev_key.unwrap().next())?,
+                            writer
+                                .take()
+                                .unwrap()
+                                .finish(prev_key.unwrap().next())
+                                .await?,
                         ));
                         writer = None;
 
@@ -3633,20 +3638,23 @@ impl Timeline {
             }
             if writer.is_none() {
                 // Create writer if not initiaized yet
-                writer = Some(DeltaLayerWriter::new(
-                    self.conf,
-                    self.timeline_id,
-                    self.tenant_id,
-                    key,
-                    if dup_end_lsn.is_valid() {
-                        // this is a layer containing slice of values of the same key
-                        debug!("Create new dup layer {}..{}", dup_start_lsn, dup_end_lsn);
-                        dup_start_lsn..dup_end_lsn
-                    } else {
-                        debug!("Create new layer {}..{}", lsn_range.start, lsn_range.end);
-                        lsn_range.clone()
-                    },
-                )?);
+                writer = Some(
+                    DeltaLayerWriter::new(
+                        self.conf,
+                        self.timeline_id,
+                        self.tenant_id,
+                        key,
+                        if dup_end_lsn.is_valid() {
+                            // this is a layer containing slice of values of the same key
+                            debug!("Create new dup layer {}..{}", dup_start_lsn, dup_end_lsn);
+                            dup_start_lsn..dup_end_lsn
+                        } else {
+                            debug!("Create new layer {}..{}", lsn_range.start, lsn_range.end);
+                            lsn_range.clone()
+                        },
+                    )
+                    .await?,
+                );
             }
 
             fail_point!("delta-layer-writer-fail-before-finish", |_| {
@@ -3655,11 +3663,11 @@ impl Timeline {
                 )))
             });
 
-            writer.as_mut().unwrap().put_value(key, lsn, value)?;
+            writer.as_mut().unwrap().put_value(key, lsn, value).await?;
             prev_key = Some(key);
         }
         if let Some(writer) = writer {
-            new_layers.push(Arc::new(writer.finish(prev_key.unwrap().next())?));
+            new_layers.push(Arc::new(writer.finish(prev_key.unwrap().next()).await?));
         }
 
         // Sync layers
diff --git a/pageserver/src/virtual_file.rs b/pageserver/src/virtual_file.rs
index 41c5eb96cf..7e9b50dfeb 100644
--- a/pageserver/src/virtual_file.rs
+++ b/pageserver/src/virtual_file.rs
@@ -326,7 +326,7 @@ impl VirtualFile {
         self.with_file("fsync", |file| file.sync_all())?
     }
 
-    pub fn metadata(&self) -> Result<fs::Metadata, Error> {
+    pub async fn metadata(&self) -> Result<fs::Metadata, Error> {
         self.with_file("metadata", |file| file.metadata())?
     }
 
@@ -407,7 +407,7 @@ impl VirtualFile {
         std::fs::remove_file(path).expect("failed to remove the virtual file");
     }
 
-    pub fn seek(&mut self, pos: SeekFrom) -> Result<u64, Error> {
+    pub async fn seek(&mut self, pos: SeekFrom) -> Result<u64, Error> {
         match pos {
             SeekFrom::Start(offset) => {
                 self.pos = offset;
@@ -619,9 +619,9 @@ mod tests {
                 MaybeVirtualFile::File(file) => file.write_all_at(buf, offset),
             }
         }
-        fn seek(&mut self, pos: SeekFrom) -> Result<u64, Error> {
+        async fn seek(&mut self, pos: SeekFrom) -> Result<u64, Error> {
             match self {
-                MaybeVirtualFile::VirtualFile(file) => file.seek(pos),
+                MaybeVirtualFile::VirtualFile(file) => file.seek(pos).await,
                 MaybeVirtualFile::File(file) => file.seek(pos),
             }
         }
@@ -712,23 +712,23 @@ mod tests {
         assert_eq!("", file_a.read_string().await?);
 
         // Test seeks.
-        assert_eq!(file_a.seek(SeekFrom::Start(1))?, 1);
+        assert_eq!(file_a.seek(SeekFrom::Start(1)).await?, 1);
         assert_eq!("oobar", file_a.read_string().await?);
 
-        assert_eq!(file_a.seek(SeekFrom::End(-2))?, 4);
+        assert_eq!(file_a.seek(SeekFrom::End(-2)).await?, 4);
         assert_eq!("ar", file_a.read_string().await?);
 
-        assert_eq!(file_a.seek(SeekFrom::Start(1))?, 1);
-        assert_eq!(file_a.seek(SeekFrom::Current(2))?, 3);
+        assert_eq!(file_a.seek(SeekFrom::Start(1)).await?, 1);
+        assert_eq!(file_a.seek(SeekFrom::Current(2)).await?, 3);
         assert_eq!("bar", file_a.read_string().await?);
 
-        assert_eq!(file_a.seek(SeekFrom::Current(-5))?, 1);
+        assert_eq!(file_a.seek(SeekFrom::Current(-5)).await?, 1);
         assert_eq!("oobar", file_a.read_string().await?);
 
         // Test erroneous seeks to before byte 0
-        assert!(file_a.seek(SeekFrom::End(-7)).is_err());
-        assert_eq!(file_a.seek(SeekFrom::Start(1))?, 1);
-        assert!(file_a.seek(SeekFrom::Current(-2)).is_err());
+        file_a.seek(SeekFrom::End(-7)).await.unwrap_err();
+        assert_eq!(file_a.seek(SeekFrom::Start(1)).await?, 1);
+        file_a.seek(SeekFrom::Current(-2)).await.unwrap_err();
 
         // the erroneous seek should have left the position unchanged
         assert_eq!("oobar", file_a.read_string().await?);
@@ -752,7 +752,7 @@ mod tests {
         // open the same file many times. The effect is the same.)
         //
         // leave file_a positioned at offset 1 before we start
-        assert_eq!(file_a.seek(SeekFrom::Start(1))?, 1);
+        assert_eq!(file_a.seek(SeekFrom::Start(1)).await?, 1);
 
         let mut vfiles = Vec::new();
         for _ in 0..100 {

From 7ceddadb3770fda09e00bd90d62bee17804d15b7 Mon Sep 17 00:00:00 2001
From: Alexander Bayandin <alexander@neon.tech>
Date: Tue, 5 Sep 2023 14:02:37 +0100
Subject: [PATCH 02/61] Merge custom extension CI jobs (#5194)

## Problem

When a remote custom extension build fails, it looks a bit confusing on
neon CI:
- `trigger-custom-extensions-build` is green
- `wait-for-extensions-build` is red
- `build-and-upload-extensions` is red

But to restart the build (to get everything green), you need to restart
the only passed `trigger-custom-extensions-build`.

## Summary of changes
- Merge `trigger-custom-extensions-build` and
`wait-for-extensions-build` jobs into
`trigger-custom-extensions-build-and-wait`
---
 .github/workflows/build_and_test.yml | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 144a96910e..6cefe5bc9d 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -899,11 +899,8 @@ jobs:
       - name: Cleanup ECR folder
         run: rm -rf ~/.ecr
 
-  trigger-custom-extensions-build:
-    runs-on: [ self-hosted, gen3, small ]
-    container:
-      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned
-      options: --init
+  trigger-custom-extensions-build-and-wait:
+    runs-on: ubuntu-latest
     needs: [ tag ]
     steps:
       - name: Set PR's status to pending and request a remote CI test
@@ -938,16 +935,11 @@ jobs:
               }
             }"
 
-  wait-for-extensions-build:
-    runs-on: ubuntu-latest
-    needs: [ trigger-custom-extensions-build ]
-
-    steps:
       - name: Wait for extension build to finish
         env:
           GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
         run: |
-          TIMEOUT=600 # 10 minutes, currently it takes ~2-3 minutes
+          TIMEOUT=1800 # 30 minutes, usually it takes ~2-3 minutes, but if runners are busy, it might take longer
           INTERVAL=15 # try each N seconds
 
           last_status="" # a variable to carry the last status of the "build-and-upload-extensions" context
@@ -984,7 +976,7 @@ jobs:
   deploy:
     runs-on: [ self-hosted, gen3, small ]
     container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest
-    needs: [ promote-images, tag, regress-tests, wait-for-extensions-build ]
+    needs: [ promote-images, tag, regress-tests, trigger-custom-extensions-build-and-wait ]
     if: ( github.ref_name == 'main' || github.ref_name == 'release' ) && github.event_name != 'workflow_dispatch'
     steps:
       - name: Fix git ownership

From 89c64e179e6051585b20035227017c9e70748279 Mon Sep 17 00:00:00 2001
From: MMeent <matthias@neon.tech>
Date: Tue, 5 Sep 2023 20:00:47 +0200
Subject: [PATCH 03/61] Fix corruption issue in Local File Cache (#5208)

Fix issue where updating the size of the Local File Cache could lead to
invalid reads:

## Problem

LFC cache can get re-enabled when lfc_max_size is set, e.g. through an
autoscaler configuration, or PostgreSQL not liking us setting the
variable.

1. initialize: LFC enabled (lfc_size_limit > 0; lfc_desc = 0)
2. Open LFC file fails, lfc_desc = -1. lfc_size_limit is set to 0;
3. lfc_size_limit is updated by autoscaling to >0
4. read() now thinks LFC is enabled (size_limit > 0) and lfc_desc is
valid, but doesn't try to read from the invalid file handle and thus
doesn't update the buffer content with the page's data, but does think
the data was read...
Any buffer we try to read from local file cache is essentially
uninitialized memory. Those are likely 0-bytes, but might potentially be
any old buffer that was previously read from or flushed to disk.

Fix this by adding a more definitive disable flag, plus better invalid state handling.
---
 pgxn/neon/file_cache.c | 31 ++++++++++++++++++-------------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/pgxn/neon/file_cache.c b/pgxn/neon/file_cache.c
index 262814c818..a16b6bca2f 100644
--- a/pgxn/neon/file_cache.c
+++ b/pgxn/neon/file_cache.c
@@ -83,11 +83,12 @@ typedef struct FileCacheControl
 } FileCacheControl;
 
 static HTAB* lfc_hash;
-static int   lfc_desc;
+static int   lfc_desc = 0;
 static LWLockId lfc_lock;
 static int   lfc_max_size;
 static int   lfc_size_limit;
 static int   lfc_free_space_watermark;
+static bool	lfc_disabled_by_failure = false;
 static char* lfc_path;
 static  FileCacheControl* lfc_ctl;
 static shmem_startup_hook_type prev_shmem_startup_hook;
@@ -96,6 +97,8 @@ static shmem_request_hook_type prev_shmem_request_hook;
 #endif
 static int   lfc_shrinking_factor; /* power of two by which local cache size will be shrinked when lfc_free_space_watermark is reached */
 
+#define DISABLE_LFC() (lfc_max_size = 0, lfc_disabled_by_failure = true, lfc_desc = -1)
+
 void FileCacheMonitorMain(Datum main_arg);
 
 static void
@@ -168,7 +171,7 @@ lfc_change_limit_hook(int newval, void *extra)
 		return;
 
 	/* Open cache file if not done yet */
-	if (lfc_desc == 0)
+	if (lfc_desc <= 0)
 	{
 		lfc_desc = BasicOpenFile(lfc_path, O_RDWR|O_CREAT);
 		if (lfc_desc < 0) {
@@ -328,7 +331,7 @@ lfc_init(void)
 							   NULL,
 							   NULL);
 
-	if (lfc_max_size == 0)
+	if (lfc_max_size == 0 || lfc_disabled_by_failure)
 		return;
 
 	if (lfc_free_space_watermark != 0)
@@ -357,7 +360,7 @@ lfc_cache_contains(RelFileNode rnode, ForkNumber forkNum, BlockNumber blkno)
 	bool found;
 	uint32 hash;
 
-	if (lfc_size_limit == 0) /* fast exit if file cache is disabled */
+	if (lfc_size_limit == 0 || lfc_disabled_by_failure) /* fast exit if file cache is disabled */
 		return false;
 
 	tag.rnode = rnode;
@@ -384,7 +387,7 @@ lfc_evict(RelFileNode rnode, ForkNumber forkNum, BlockNumber blkno)
 	int chunk_offs = blkno & (BLOCKS_PER_CHUNK-1);
 	uint32 hash;
 
-	if (lfc_size_limit == 0) /* fast exit if file cache is disabled */
+	if (lfc_size_limit == 0 || lfc_disabled_by_failure) /* fast exit if file cache is disabled */
 		return;
 
 	INIT_BUFFERTAG(tag, rnode, forkNum, (blkno & ~(BLOCKS_PER_CHUNK-1)));
@@ -455,7 +458,7 @@ lfc_read(RelFileNode rnode, ForkNumber forkNum, BlockNumber blkno,
 	bool result = true;
 	uint32 hash;
 
-	if (lfc_size_limit == 0) /* fast exit if file cache is disabled */
+	if (lfc_size_limit == 0 || lfc_disabled_by_failure) /* fast exit if file cache is disabled */
 		return false;
 
 	tag.rnode = rnode;
@@ -477,23 +480,25 @@ lfc_read(RelFileNode rnode, ForkNumber forkNum, BlockNumber blkno,
 	LWLockRelease(lfc_lock);
 
 	/* Open cache file if not done yet */
-	if (lfc_desc == 0)
+	if (lfc_desc <= 0)
 	{
 		lfc_desc = BasicOpenFile(lfc_path, O_RDWR|O_CREAT);
+
 		if (lfc_desc < 0) {
 			elog(LOG, "Failed to open file cache %s: %m", lfc_path);
-			lfc_size_limit = 0; /* disable file cache */
+			DISABLE_LFC();
 			result = false;
 		}
 	}
 
+
 	if (lfc_desc > 0)
 	{
 		rc = pread(lfc_desc, buffer, BLCKSZ, ((off_t)entry->offset*BLOCKS_PER_CHUNK + chunk_offs)*BLCKSZ);
 		if (rc != BLCKSZ)
 		{
 			elog(INFO, "Failed to read file cache: %m");
-			lfc_size_limit = 0; /* disable file cache */
+			DISABLE_LFC();
 			result = false;
 		}
 	}
@@ -523,7 +528,7 @@ lfc_write(RelFileNode rnode, ForkNumber forkNum, BlockNumber blkno,
 	int chunk_offs = blkno & (BLOCKS_PER_CHUNK-1);
 	uint32 hash;
 
-	if (lfc_size_limit == 0) /* fast exit if file cache is disabled */
+	if (lfc_size_limit == 0 || lfc_disabled_by_failure) /* fast exit if file cache is disabled */
 		return;
 
 	tag.rnode = rnode;
@@ -570,12 +575,12 @@ lfc_write(RelFileNode rnode, ForkNumber forkNum, BlockNumber blkno,
 	LWLockRelease(lfc_lock);
 
 	/* Open cache file if not done yet */
-	if (lfc_desc == 0)
+	if (lfc_desc <= 0)
 	{
 		lfc_desc = BasicOpenFile(lfc_path, O_RDWR|O_CREAT);
 		if (lfc_desc < 0) {
 			elog(WARNING, "Failed to open file cache %s: %m, disabling file cache", lfc_path);
-			lfc_size_limit = 0; /* disable file cache */
+			DISABLE_LFC(); /* disable file cache */
 		}
 	}
 	if (lfc_desc > 0)
@@ -584,7 +589,7 @@ lfc_write(RelFileNode rnode, ForkNumber forkNum, BlockNumber blkno,
 		if (rc != BLCKSZ)
 		{
 			elog(WARNING, "Failed to write file cache: %m, disabling file cache");
-			lfc_size_limit = 0; /* disable file cache */
+			DISABLE_LFC(); /* disable file cache */
 		}
 	}
 	/* Place entry to the head of LRU list */

From c222320a2affb4beda089867878cc06040e2769a Mon Sep 17 00:00:00 2001
From: Alexander Bayandin <alexander@neon.tech>
Date: Wed, 6 Sep 2023 00:48:15 +0100
Subject: [PATCH 04/61] Generate lcov coverage report (#4999)

## Problem

We want to display coverage information for each PR.

- an example of a full coverage report:
https://neon-github-public-dev.s3.amazonaws.com/code-coverage/abea64800fb390c32a3efe6795d53d8621115c83/lcov/index.html
- an example of GitHub auto-comment with coverage information:
https://github.com/neondatabase/neon/pull/4999#issuecomment-1679344658

## Summary of changes
- Use
patched[*](https://github.com/neondatabase/lcov/commit/426e7e7a22f669da54278e9b55e6d8caabd00af0)
lcov to generate coverage report
- Upload HTML coverage report to S3
- `scripts/comment-test-report.js`: add coverage information
---
 .github/workflows/build_and_test.yml | 68 ++++++++++++++++++++++------
 scripts/comment-test-report.js       | 40 +++++++++++++++-
 2 files changed, 91 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 6cefe5bc9d..920882517b 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -422,7 +422,7 @@ jobs:
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init
-    needs: [ regress-tests, benchmarks ]
+    needs: [ regress-tests, coverage-report, benchmarks ]
     if: ${{ !cancelled() }}
 
     steps:
@@ -449,12 +449,18 @@ jobs:
               reportJsonUrl: "${{ steps.create-allure-report.outputs.report-json-url }}",
             }
 
+            const coverage = {
+              coverageUrl: "${{ needs.coverage-report.outputs.coverage-html }}",
+              summaryJsonUrl: "${{ needs.coverage-report.outputs.coverage-json }}",
+            }
+
             const script = require("./scripts/comment-test-report.js")
             await script({
               github,
               context,
               fetch,
               report,
+              coverage,
             })
 
   coverage-report:
@@ -467,24 +473,15 @@ jobs:
       fail-fast: false
       matrix:
         build_type: [ debug ]
+    outputs:
+        coverage-html: ${{ steps.upload-coverage-report-new.outputs.report-url }}
+        coverage-json: ${{ steps.upload-coverage-report-new.outputs.summary-json }}
     steps:
       - name: Checkout
         uses: actions/checkout@v3
         with:
           submodules: true
-          fetch-depth: 1
-
-#      Disabled for now
-#      - name: Restore cargo deps cache
-#        id: cache_cargo
-#        uses: actions/cache@v3
-#        with:
-#          path: |
-#            ~/.cargo/registry/
-#            !~/.cargo/registry/src
-#            ~/.cargo/git/
-#            target/
-#          key: v1-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('Cargo.lock') }}
+          fetch-depth: 0
 
       - name: Get Neon artifact
         uses: ./.github/actions/download
@@ -527,13 +524,45 @@ jobs:
           REPORT_URL=https://${BUCKET}.s3.amazonaws.com/code-coverage/${COMMIT_SHA}/index.html
           echo "report-url=${REPORT_URL}" >> $GITHUB_OUTPUT
 
+      - name: Build coverage report NEW
+        id: upload-coverage-report-new
+        env:
+          BUCKET: neon-github-public-dev
+          COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+        run: |
+          BASELINE="$(git merge-base HEAD origin/main)"
+          CURRENT="${COMMIT_SHA}"
+
+          cp /tmp/coverage/report/lcov.info ./${CURRENT}.info
+
+          GENHTML_ARGS="--ignore-errors path,unmapped,empty --synthesize-missing --demangle-cpp rustfilt --output-directory lcov-html ${CURRENT}.info"
+
+          # Use differential coverage if the baseline coverage exists.
+          # It can be missing if the coverage repoer wasn't uploaded yet or tests has failed on BASELINE commit.
+          if aws s3 cp --only-show-errors s3://${BUCKET}/code-coverage/${BASELINE}/lcov.info ./${BASELINE}.info; then
+            git diff ${BASELINE} ${CURRENT} -- '*.rs' > baseline-current.diff
+
+            GENHTML_ARGS="--baseline-file ${BASELINE}.info --diff-file baseline-current.diff ${GENHTML_ARGS}"
+          fi
+
+          genhtml ${GENHTML_ARGS}
+
+          aws s3 cp --only-show-errors --recursive ./lcov-html/ s3://${BUCKET}/code-coverage/${COMMIT_SHA}/lcov
+
+          REPORT_URL=https://${BUCKET}.s3.amazonaws.com/code-coverage/${COMMIT_SHA}/lcov/index.html
+          echo "report-url=${REPORT_URL}" >> $GITHUB_OUTPUT
+
+          REPORT_URL=https://${BUCKET}.s3.amazonaws.com/code-coverage/${COMMIT_SHA}/lcov/summary.json
+          echo "summary-json=${REPORT_URL}" >> $GITHUB_OUTPUT
+
       - uses: actions/github-script@v6
         env:
           REPORT_URL: ${{ steps.upload-coverage-report.outputs.report-url }}
+          REPORT_URL_NEW: ${{ steps.upload-coverage-report-new.outputs.report-url }}
           COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
         with:
           script: |
-            const { REPORT_URL, COMMIT_SHA } = process.env
+            const { REPORT_URL, REPORT_URL_NEW, COMMIT_SHA } = process.env
 
             await github.rest.repos.createCommitStatus({
               owner: context.repo.owner,
@@ -544,6 +573,15 @@ jobs:
               context: 'Code coverage report',
             })
 
+            await github.rest.repos.createCommitStatus({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              sha: `${COMMIT_SHA}`,
+              state: 'success',
+              target_url: `${REPORT_URL_NEW}`,
+              context: 'Code coverage report NEW',
+            })
+
   trigger-e2e-tests:
     runs-on: [ self-hosted, gen3, small ]
     container:
diff --git a/scripts/comment-test-report.js b/scripts/comment-test-report.js
index 1410b8a0ca..f62d923dc4 100755
--- a/scripts/comment-test-report.js
+++ b/scripts/comment-test-report.js
@@ -18,6 +18,10 @@
 //           reportUrl: "...",
 //           reportJsonUrl: "...",
 //         },
+//         coverage: {
+//           coverageUrl: "...",
+//           summaryJsonUrl: "...",
+//         }
 //       })
 //
 
@@ -183,7 +187,24 @@ const reportSummary = async (params) => {
     return summary
 }
 
-module.exports = async ({ github, context, fetch, report }) => {
+const parseCoverageSummary = async ({ summaryJsonUrl, coverageUrl, fetch }) => {
+    let summary = `### Code coverage [full report](${coverageUrl})\n`
+
+    const coverage = await (await fetch(summaryJsonUrl)).json()
+    for (const covType of Object.keys(coverage).sort()) {
+        if (!coverage.hasOwnProperty(covType)) {
+            continue
+        }
+
+        summary += `- \`${covType}s\`: \`${coverage[covType]["_summary"]}\`\n`
+    }
+
+    summary += `\n___\n`
+
+    return summary
+}
+
+module.exports = async ({ github, context, fetch, report, coverage }) => {
     // Marker to find the comment in the subsequent runs
     const startMarker = `<!--AUTOMATIC COMMENT START #${context.payload.number}-->`
     // If we run the script in the PR or in the branch (main/release/...)
@@ -204,7 +225,6 @@ module.exports = async ({ github, context, fetch, report }) => {
     }
 
     const {reportUrl, reportJsonUrl} = report
-
     if (reportUrl && reportJsonUrl) {
         try {
             const parsed = await parseReportJson({ reportJsonUrl, fetch })
@@ -223,6 +243,22 @@ module.exports = async ({ github, context, fetch, report }) => {
     } else {
         commentBody += `#### No tests were run or test report is not available\n`
     }
+
+    const { coverageUrl, summaryJsonUrl } = coverage
+    if (coverageUrl && summaryJsonUrl) {
+        try {
+            commentBody += await parseCoverageSummary({ summaryJsonUrl, coverageUrl, fetch })
+        } catch (error) {
+            commentBody += `### [full report](${coverageUrl})\n___\n`
+            commentBody += `#### Failed to create a coverage summary for the test run: \n`
+            commentBody += "```\n"
+            commentBody += `${error.stack}\n`
+            commentBody += "```\n"
+        }
+    } else {
+        commentBody += `#### Test coverage report is not avaibale\n`
+    }
+
     commentBody += autoupdateNotice
 
     let createCommentFn, listCommentsFn, updateCommentFn, issueNumberOrSha

From dbfb4ea7b82fd1cbdc79dccbc1a5968ca816da37 Mon Sep 17 00:00:00 2001
From: Alexander Bayandin <alexander@neon.tech>
Date: Wed, 6 Sep 2023 10:06:55 +0100
Subject: [PATCH 05/61] Make CI more friendly for external contributors. Second
 try (#5210)

## Problem

`approved-for-ci-run` label logic doesn't work as expected:
- https://github.com/neondatabase/neon/pull/4722#issuecomment-1636742145
- https://github.com/neondatabase/neon/pull/4722#issuecomment-1636755394

Continuation of https://github.com/neondatabase/neon/pull/4663
Closes #2222 (hopefully)

## Summary of changes
- Create a twin PR automatically
- Allow `GITHUB_TOKEN` to manipulate with labels
---
 .github/workflows/approved-for-ci-run.yml | 32 ++++++++++++++++++-----
 .github/workflows/build_and_test.yml      |  1 -
 .github/workflows/neon_extra_builds.yml   |  1 -
 3 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/approved-for-ci-run.yml b/.github/workflows/approved-for-ci-run.yml
index ac9e908c09..c2be9adea2 100644
--- a/.github/workflows/approved-for-ci-run.yml
+++ b/.github/workflows/approved-for-ci-run.yml
@@ -3,6 +3,8 @@ name: Handle `approved-for-ci-run` label
 
 on:
   pull_request:
+    branches:
+      - main
     types:
       # Default types that triggers a workflow ([1]):
       # - [1] https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request
@@ -18,29 +20,34 @@ env:
   GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   PR_NUMBER: ${{ github.event.pull_request.number }}
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+
 jobs:
   remove-label:
     # Remove `approved-for-ci-run` label if the workflow is triggered by changes in a PR.
     # The PR should be reviewed and labelled manually again.
 
-    runs-on: [ ubuntu-latest ]
-
     if: |
       contains(fromJSON('["opened", "synchronize", "reopened", "closed"]'), github.event.action) &&
       contains(github.event.pull_request.labels.*.name, 'approved-for-ci-run')
 
+    runs-on: ubuntu-latest
+    permissions: write-all
+
     steps:
       - run: gh pr --repo "${GITHUB_REPOSITORY}" edit "${PR_NUMBER}" --remove-label "approved-for-ci-run"
 
-  create-branch:
-    # Create a local branch for an `approved-for-ci-run` labelled PR to run CI pipeline in it.
-
-    runs-on: [ ubuntu-latest ]
+  create-or-update-pr-for-ci-run:
+    # Create local PR for an `approved-for-ci-run` labelled PR to run CI pipeline in it.
 
     if: |
       github.event.action == 'labeled' &&
       contains(github.event.pull_request.labels.*.name, 'approved-for-ci-run')
 
+    runs-on: ubuntu-latest
+    permissions: write-all
+
     steps:
       - run: gh pr --repo "${GITHUB_REPOSITORY}" edit "${PR_NUMBER}" --remove-label "approved-for-ci-run"
 
@@ -53,3 +60,16 @@ jobs:
       - run: git checkout -b "ci-run/pr-${PR_NUMBER}"
 
       - run: git push --force origin "ci-run/pr-${PR_NUMBER}"
+
+      - name: Create a Pull Request for CI run (if required)
+        run: |
+          HEAD="ci-run/pr-${PR_NUMBER}"
+
+          ALREADY_CREATED=$(gh pr --repo "${GITHUB_REPOSITORY}" list --head "${HEAD}" --base main --json number --jq '.[].number')
+          if [ -z "${ALREADY_CREATED}" ]; then
+            gh pr --repo "${GITHUB_REPOSITORY}" create  --title "[DO NOT MERGE] CI run for PR #${PR_NUMBER}" \
+                                                        --body "Ref #${PR_NUMBER}" \
+                                                        --head "${HEAD}" \
+                                                        --base main \
+                                                        --draft
+          fi
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 920882517b..d03f8c4365 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -5,7 +5,6 @@ on:
     branches:
       - main
       - release
-      - ci-run/pr-*
   pull_request:
 
 defaults:
diff --git a/.github/workflows/neon_extra_builds.yml b/.github/workflows/neon_extra_builds.yml
index a21ddb0414..f4e9e27e2b 100644
--- a/.github/workflows/neon_extra_builds.yml
+++ b/.github/workflows/neon_extra_builds.yml
@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - main
-      - ci-run/pr-*
   pull_request:
 
 defaults:

From 15ff4e5fd11849538c20d6da64af4a373fb9379a Mon Sep 17 00:00:00 2001
From: Alexander Bayandin <alexander@neon.tech>
Date: Wed, 6 Sep 2023 11:14:07 +0100
Subject: [PATCH 06/61] approved-for-ci-run.yml: trigger on pull_request_target
 (#5212)

## Problem

Continuation of #4663, #5210

We're still getting an error:
```
GraphQL: Resource not accessible by integration (removeLabelsFromLabelable)
```

## Summary of changes
- trigger `approved-for-ci-run.yml` workflow on `pull_request_target`
instead of `pull_request`
---
 .github/workflows/approved-for-ci-run.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/approved-for-ci-run.yml b/.github/workflows/approved-for-ci-run.yml
index c2be9adea2..77cedae375 100644
--- a/.github/workflows/approved-for-ci-run.yml
+++ b/.github/workflows/approved-for-ci-run.yml
@@ -2,7 +2,7 @@ name: Handle `approved-for-ci-run` label
 # This workflow helps to run CI pipeline for PRs made by external contributors (from forks).
 
 on:
-  pull_request:
+  pull_request_target:
     branches:
       - main
     types:

From 88b1ac48bd6d8acfc85a34f436fc9ce8af63d8e1 Mon Sep 17 00:00:00 2001
From: Vadim Kharitonov <vadim2404@users.noreply.github.com>
Date: Wed, 6 Sep 2023 12:17:52 +0200
Subject: [PATCH 07/61] Create Release PR at 7:00 UTC every Tuesday (#5213)

---
 .github/workflows/release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 595ee05514..17a43fe8c1 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -2,7 +2,7 @@ name: Create Release Branch
 
 on:
   schedule:
-    - cron: '0 10 * * 2'
+    - cron: '0 7 * * 2'
   workflow_dispatch:
 
 jobs:

From 4fec48f2b5dd35a478151e364212877b68e9732d Mon Sep 17 00:00:00 2001
From: duguorong009 <80258679+duguorong009@users.noreply.github.com>
Date: Wed, 6 Sep 2023 18:19:19 +0800
Subject: [PATCH 08/61] chore(pageserver): remove unnecessary logging in tenant
 task loops (#5188)

Fixes #3830 by adding the `#[cfg(not(feature = "testing"))]` attribute
to unnecessary loggings in `pageserver/src/tenant/tasks.rs`.

Co-authored-by: Joonas Koivunen <joonas@neon.tech>
---
 pageserver/src/tenant/tasks.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pageserver/src/tenant/tasks.rs b/pageserver/src/tenant/tasks.rs
index c067a84471..df3ffd08d3 100644
--- a/pageserver/src/tenant/tasks.rs
+++ b/pageserver/src/tenant/tasks.rs
@@ -102,6 +102,7 @@ async fn compaction_loop(tenant: Arc<Tenant>, cancel: CancellationToken) {
             let started_at = Instant::now();
 
             let sleep_duration = if period == Duration::ZERO {
+                #[cfg(not(feature = "testing"))]
                 info!("automatic compaction is disabled");
                 // check again in 10 seconds, in case it's been enabled again.
                 Duration::from_secs(10)
@@ -166,6 +167,7 @@ async fn gc_loop(tenant: Arc<Tenant>, cancel: CancellationToken) {
 
             let gc_horizon = tenant.get_gc_horizon();
             let sleep_duration = if period == Duration::ZERO || gc_horizon == 0 {
+                #[cfg(not(feature = "testing"))]
                 info!("automatic GC is disabled");
                 // check again in 10 seconds, in case it's been enabled again.
                 Duration::from_secs(10)

From 8e25d3e79e467fff51f09bf027bed1701fe9177b Mon Sep 17 00:00:00 2001
From: Alexander Bayandin <alexander@neon.tech>
Date: Wed, 6 Sep 2023 11:22:57 +0100
Subject: [PATCH 09/61] test_runner: add scale parameter to tpc-h tests (#5079)

## Problem

It's hard to find out which DB size we use for OLAP benchmarks (TPC-H in
particular).
This PR adds handling of `TEST_OLAP_SCALE` env var, which is get added
to a test name as a parameter.

This is required for performing larger periodic benchmarks.

## Summary of changes
- Handle `TEST_OLAP_SCALE` in
`test_runner/performance/test_perf_olap.py`
- Set `TEST_OLAP_SCALE` in `.github/workflows/benchmarking.yml` to a
TPC-H scale
---
 .github/workflows/benchmarking.yml        | 66 +++++++++++++++++++----
 test_runner/performance/test_perf_olap.py | 18 +++++--
 2 files changed, 71 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml
index 172b904331..bea88e67a5 100644
--- a/.github/workflows/benchmarking.yml
+++ b/.github/workflows/benchmarking.yml
@@ -117,6 +117,7 @@ jobs:
     outputs:
       pgbench-compare-matrix: ${{ steps.pgbench-compare-matrix.outputs.matrix }}
       olap-compare-matrix: ${{ steps.olap-compare-matrix.outputs.matrix }}
+      tpch-compare-matrix: ${{ steps.tpch-compare-matrix.outputs.matrix }}
 
     steps:
     - name: Generate matrix for pgbench benchmark
@@ -158,6 +159,25 @@ jobs:
 
         echo "matrix=$(echo $matrix | jq --compact-output '.')" >> $GITHUB_OUTPUT
 
+    - name: Generate matrix for TPC-H benchmarks
+      id: tpch-compare-matrix
+      run: |
+        matrix='{
+          "platform": [
+            "neon-captest-reuse"
+          ],
+          "scale": [
+            "10"
+          ]
+        }'
+
+        if [ "$(date +%A)" = "Saturday" ]; then
+          matrix=$(echo $matrix | jq '.include += [{ "platform": "rds-postgres", "scale": "10" },
+                                                   { "platform": "rds-aurora",   "scale": "10" }]')
+        fi
+
+        echo "matrix=$(echo $matrix | jq --compact-output '.')" >> $GITHUB_OUTPUT
+
   pgbench-compare:
     needs: [ generate-matrices ]
 
@@ -233,7 +253,11 @@ jobs:
 
         echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
 
-        psql ${CONNSTR} -c "SELECT version();"
+        QUERY="SELECT version();"
+        if [ "${PLATFORM}" = "neon"* ]; then
+          QUERY="${QUERY} SHOW neon.tenant_id; SHOW neon.timeline_id;"
+        fi
+        psql ${CONNSTR} -c "${QUERY}"
 
     - name: Benchmark init
       uses: ./.github/actions/run-python-test-set
@@ -358,7 +382,11 @@ jobs:
 
         echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
 
-        psql ${CONNSTR} -c "SELECT version();"
+        QUERY="SELECT version();"
+        if [ "${PLATFORM}" = "neon"* ]; then
+          QUERY="${QUERY} SHOW neon.tenant_id; SHOW neon.timeline_id;"
+        fi
+        psql ${CONNSTR} -c "${QUERY}"
 
     - name: ClickBench benchmark
       uses: ./.github/actions/run-python-test-set
@@ -372,6 +400,7 @@ jobs:
         VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
         PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
         BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
+        TEST_OLAP_SCALE: 10
 
     - name: Create Allure report
       if: ${{ !cancelled() }}
@@ -398,7 +427,7 @@ jobs:
 
     strategy:
       fail-fast: false
-      matrix: ${{ fromJson(needs.generate-matrices.outputs.olap-compare-matrix) }}
+      matrix: ${{ fromJson(needs.generate-matrices.outputs.tpch-compare-matrix) }}
 
     env:
       POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
@@ -407,6 +436,7 @@ jobs:
       BUILD_TYPE: remote
       SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
       PLATFORM: ${{ matrix.platform }}
+      TEST_OLAP_SCALE: ${{ matrix.scale }}
 
     runs-on: [ self-hosted, us-east-2, x64 ]
     container:
@@ -428,18 +458,17 @@ jobs:
         ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version
         echo "${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin" >> $GITHUB_PATH
 
-    - name: Set up Connection String
-      id: set-up-connstr
+    - name: Get Connstring Secret Name
       run: |
         case "${PLATFORM}" in
           neon-captest-reuse)
-            CONNSTR=${{ secrets.BENCHMARK_CAPTEST_TPCH_S10_CONNSTR }}
+            ENV_PLATFORM=CAPTEST_TPCH
             ;;
           rds-aurora)
-            CONNSTR=${{ secrets.BENCHMARK_RDS_AURORA_TPCH_S10_CONNSTR }}
+            ENV_PLATFORM=RDS_AURORA_TPCH
             ;;
           rds-postgres)
-            CONNSTR=${{ secrets.BENCHMARK_RDS_POSTGRES_TPCH_S10_CONNSTR }}
+            ENV_PLATFORM=RDS_AURORA_TPCH
             ;;
           *)
             echo >&2 "Unknown PLATFORM=${PLATFORM}. Allowed only 'neon-captest-reuse', 'rds-aurora', or 'rds-postgres'"
@@ -447,9 +476,21 @@ jobs:
             ;;
         esac
 
+        CONNSTR_SECRET_NAME="BENCHMARK_${ENV_PLATFORM}_S${SCALE}_CONNSTR"
+        echo "CONNSTR_SECRET_NAME=${CONNSTR_SECRET_NAME}" >> $GITHUB_ENV
+
+    - name: Set up Connection String
+      id: set-up-connstr
+      run: |
+        CONNSTR=${{ secrets[env.CONNSTR_SECRET_NAME] }}
+
         echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
 
-        psql ${CONNSTR} -c "SELECT version();"
+        QUERY="SELECT version();"
+        if [ "${PLATFORM}" = "neon"* ]; then
+          QUERY="${QUERY} SHOW neon.tenant_id; SHOW neon.timeline_id;"
+        fi
+        psql ${CONNSTR} -c "${QUERY}"
 
     - name: Run TPC-H benchmark
       uses: ./.github/actions/run-python-test-set
@@ -463,6 +504,7 @@ jobs:
         VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
         PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
         BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
+        TEST_OLAP_SCALE: ${{ matrix.scale }}
 
     - name: Create Allure report
       if: ${{ !cancelled() }}
@@ -534,7 +576,11 @@ jobs:
 
         echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
 
-        psql ${CONNSTR} -c "SELECT version();"
+        QUERY="SELECT version();"
+        if [ "${PLATFORM}" = "neon"* ]; then
+          QUERY="${QUERY} SHOW neon.tenant_id; SHOW neon.timeline_id;"
+        fi
+        psql ${CONNSTR} -c "${QUERY}"
 
     - name: Run user examples
       uses: ./.github/actions/run-python-test-set
diff --git a/test_runner/performance/test_perf_olap.py b/test_runner/performance/test_perf_olap.py
index 39aafa80df..0f7615f7ed 100644
--- a/test_runner/performance/test_perf_olap.py
+++ b/test_runner/performance/test_perf_olap.py
@@ -1,5 +1,6 @@
+import os
 from dataclasses import dataclass
-from typing import Dict, Tuple
+from typing import Dict, List, Tuple
 
 import pytest
 from _pytest.mark import ParameterSet
@@ -78,6 +79,15 @@ QUERIES: Tuple[LabelledQuery, ...] = (
 )
 
 
+def get_scale() -> List[str]:
+    # We parametrize each tpc-h and clickbench test with scale
+    # to distinguish them from each other, but don't really use it inside.
+    # Databases are pre-created and passed through BENCHMARK_CONNSTR env variable.
+
+    scale = os.getenv("TEST_OLAP_SCALE", "noscale")
+    return [scale]
+
+
 def run_psql(env: RemoteCompare, labelled_query: LabelledQuery, times: int) -> None:
     # prepare connstr:
     # - cut out password from connstr to pass it via env
@@ -100,9 +110,10 @@ def run_psql(env: RemoteCompare, labelled_query: LabelledQuery, times: int) -> N
             env.pg_bin.run_capture(["psql", connstr, "-c", query], env=environ)
 
 
+@pytest.mark.parametrize("scale", get_scale())
 @pytest.mark.parametrize("query", QUERIES)
 @pytest.mark.remote_cluster
-def test_clickbench(query: LabelledQuery, remote_compare: RemoteCompare):
+def test_clickbench(query: LabelledQuery, remote_compare: RemoteCompare, scale: str):
     """
     An OLAP-style ClickHouse benchmark
 
@@ -128,9 +139,10 @@ def tpch_queuies() -> Tuple[ParameterSet, ...]:
     )
 
 
+@pytest.mark.parametrize("scale", get_scale())
 @pytest.mark.parametrize("query", tpch_queuies())
 @pytest.mark.remote_cluster
-def test_tpch(query: LabelledQuery, remote_compare: RemoteCompare):
+def test_tpch(query: LabelledQuery, remote_compare: RemoteCompare, scale: str):
     """
     TCP-H Benchmark
 

From 743933176ee7e9cf0011b734c806219b599e1b26 Mon Sep 17 00:00:00 2001
From: John Spray <john@neon.tech>
Date: Wed, 6 Sep 2023 11:55:24 +0100
Subject: [PATCH 10/61] scrubber: add `scan-metadata` and hook into integration
 tests (#5176)

## Problem

- Scrubber's `tidy` command requires presence of a control plane
- Scrubber has no tests at all

## Summary of changes

- Add re-usable async streams for reading metadata from a bucket
- Add a `scan-metadata` command that reads from those streams and calls
existing `checks.rs` code to validate metadata, then returns a summary
struct for the bucket. Command returns nonzero status if errors are
found.
- Add an `enable_scrub_on_exit()` function to NeonEnvBuilder so that
tests using remote storage can request to have the scrubber run after
they finish
- Enable remote storarge and scrub_on_exit in test_pageserver_restart
and test_pageserver_chaos

This is a "toe in the water" of the overall space of validating the
scrubber. Later, we should:
- Enable scrubbing at end of tests using remote storage by default
- Make the success condition stricter than "no errors": tests should
declare what tenants+timelines they expect to see in the bucket (or
sniff these from the functions tests use to create them) and we should
require that the scrubber reports on these particular tenants/timelines.

The `tidy` command is untouched in this PR, but it should be refactored
later to use similar async streaming interface instead of the current
batch-reading approach (the streams are faster with large buckets), and
to also be covered by some tests.


---------

Co-authored-by: Joonas Koivunen <joonas@neon.tech>
Co-authored-by: Alexander Bayandin <alexander@neon.tech>
Co-authored-by: Christian Schwarz <christian@neon.tech>
Co-authored-by: Conrad Ludgate <conrad@neon.tech>
---
 Cargo.lock                                    |  36 ++-
 pageserver/ctl/src/layer_map_analyzer.rs      |   5 +-
 pageserver/ctl/src/layers.rs                  |   9 +-
 pageserver/src/config.rs                      |   5 +-
 pageserver/src/tenant.rs                      |   3 +
 pageserver/src/virtual_file.rs                |   3 +-
 s3_scrubber/Cargo.toml                        |  10 +-
 s3_scrubber/src/checks.rs                     | 189 +++++++-------
 .../delete_batch_producer/timeline_batch.rs   |   2 +-
 s3_scrubber/src/lib.rs                        | 162 +++++++++---
 s3_scrubber/src/main.rs                       |  97 +++-----
 s3_scrubber/src/metadata_stream.rs            | 106 ++++++++
 s3_scrubber/src/s3_deletion.rs                |   8 +-
 s3_scrubber/src/scan_metadata.rs              | 234 ++++++++++++++++++
 test_runner/fixtures/neon_fixtures.py         |  76 +++++-
 test_runner/fixtures/remote_storage.py        |  13 +
 test_runner/fixtures/utils.py                 |  87 ++++++-
 .../regress/test_pageserver_restart.py        |   7 +
 18 files changed, 814 insertions(+), 238 deletions(-)
 create mode 100644 s3_scrubber/src/metadata_stream.rs
 create mode 100644 s3_scrubber/src/scan_metadata.rs

diff --git a/Cargo.lock b/Cargo.lock
index 98cf23c620..503c944aff 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -213,17 +213,6 @@ dependencies = [
  "critical-section",
 ]
 
-[[package]]
-name = "atty"
-version = "0.2.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
-dependencies = [
- "hermit-abi 0.1.19",
- "libc",
- "winapi",
-]
-
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -1786,15 +1775,6 @@ version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
 
-[[package]]
-name = "hermit-abi"
-version = "0.1.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "hermit-abi"
 version = "0.2.6"
@@ -1825,6 +1805,16 @@ version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46"
 
+[[package]]
+name = "histogram"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e673d137229619d5c2c8903b6ed5852b43636c0017ff2e66b1aafb8ccf04b80b"
+dependencies = [
+ "serde",
+ "thiserror",
+]
+
 [[package]]
 name = "hmac"
 version = "0.12.1"
@@ -3738,7 +3728,7 @@ name = "s3_scrubber"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "atty",
+ "async-stream",
  "aws-config",
  "aws-sdk-s3",
  "aws-smithy-http",
@@ -3749,7 +3739,10 @@ dependencies = [
  "clap",
  "crc32c",
  "either",
+ "futures-util",
  "hex",
+ "histogram",
+ "itertools",
  "pageserver",
  "rand",
  "reqwest",
@@ -3759,6 +3752,7 @@ dependencies = [
  "thiserror",
  "tokio",
  "tokio-rustls",
+ "tokio-stream",
  "tracing",
  "tracing-appender",
  "tracing-subscriber",
diff --git a/pageserver/ctl/src/layer_map_analyzer.rs b/pageserver/ctl/src/layer_map_analyzer.rs
index 29bd6ce598..32d0d1bed2 100644
--- a/pageserver/ctl/src/layer_map_analyzer.rs
+++ b/pageserver/ctl/src/layer_map_analyzer.rs
@@ -3,6 +3,7 @@
 //! Currently it only analyzes holes, which are regions within the layer range that the layer contains no updates for. In the future it might do more analysis (maybe key quantiles?) but it should never return sensitive data.
 
 use anyhow::Result;
+use pageserver::tenant::{TENANTS_SEGMENT_NAME, TIMELINES_SEGMENT_NAME};
 use std::cmp::Ordering;
 use std::collections::BinaryHeap;
 use std::ops::Range;
@@ -142,12 +143,12 @@ pub(crate) async fn main(cmd: &AnalyzeLayerMapCmd) -> Result<()> {
     let mut total_delta_layers = 0usize;
     let mut total_image_layers = 0usize;
     let mut total_excess_layers = 0usize;
-    for tenant in fs::read_dir(storage_path.join("tenants"))? {
+    for tenant in fs::read_dir(storage_path.join(TENANTS_SEGMENT_NAME))? {
         let tenant = tenant?;
         if !tenant.file_type()?.is_dir() {
             continue;
         }
-        for timeline in fs::read_dir(tenant.path().join("timelines"))? {
+        for timeline in fs::read_dir(tenant.path().join(TIMELINES_SEGMENT_NAME))? {
             let timeline = timeline?;
             if !timeline.file_type()?.is_dir() {
                 continue;
diff --git a/pageserver/ctl/src/layers.rs b/pageserver/ctl/src/layers.rs
index 608b3cecd6..ff2044653a 100644
--- a/pageserver/ctl/src/layers.rs
+++ b/pageserver/ctl/src/layers.rs
@@ -5,6 +5,7 @@ use clap::Subcommand;
 use pageserver::tenant::block_io::BlockCursor;
 use pageserver::tenant::disk_btree::DiskBtreeReader;
 use pageserver::tenant::storage_layer::delta_layer::{BlobRef, Summary};
+use pageserver::tenant::{TENANTS_SEGMENT_NAME, TIMELINES_SEGMENT_NAME};
 use pageserver::{page_cache, virtual_file};
 use pageserver::{
     repository::{Key, KEY_SIZE},
@@ -80,13 +81,13 @@ async fn read_delta_file(path: impl AsRef<Path>) -> Result<()> {
 pub(crate) async fn main(cmd: &LayerCmd) -> Result<()> {
     match cmd {
         LayerCmd::List { path } => {
-            for tenant in fs::read_dir(path.join("tenants"))? {
+            for tenant in fs::read_dir(path.join(TENANTS_SEGMENT_NAME))? {
                 let tenant = tenant?;
                 if !tenant.file_type()?.is_dir() {
                     continue;
                 }
                 println!("tenant {}", tenant.file_name().to_string_lossy());
-                for timeline in fs::read_dir(tenant.path().join("timelines"))? {
+                for timeline in fs::read_dir(tenant.path().join(TIMELINES_SEGMENT_NAME))? {
                     let timeline = timeline?;
                     if !timeline.file_type()?.is_dir() {
                         continue;
@@ -101,9 +102,9 @@ pub(crate) async fn main(cmd: &LayerCmd) -> Result<()> {
             timeline,
         } => {
             let timeline_path = path
-                .join("tenants")
+                .join(TENANTS_SEGMENT_NAME)
                 .join(tenant)
-                .join("timelines")
+                .join(TIMELINES_SEGMENT_NAME)
                 .join(timeline);
             let mut idx = 0;
             for layer in fs::read_dir(timeline_path)? {
diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs
index 5394f17398..ab485c969d 100644
--- a/pageserver/src/config.rs
+++ b/pageserver/src/config.rs
@@ -32,7 +32,8 @@ use crate::disk_usage_eviction_task::DiskUsageEvictionTaskConfig;
 use crate::tenant::config::TenantConf;
 use crate::tenant::config::TenantConfOpt;
 use crate::tenant::{
-    TENANT_ATTACHING_MARKER_FILENAME, TENANT_DELETED_MARKER_FILE_NAME, TIMELINES_SEGMENT_NAME,
+    TENANTS_SEGMENT_NAME, TENANT_ATTACHING_MARKER_FILENAME, TENANT_DELETED_MARKER_FILE_NAME,
+    TIMELINES_SEGMENT_NAME,
 };
 use crate::{
     IGNORED_TENANT_FILE_NAME, METADATA_FILE_NAME, TENANT_CONFIG_NAME, TIMELINE_DELETE_MARK_SUFFIX,
@@ -563,7 +564,7 @@ impl PageServerConf {
     //
 
     pub fn tenants_path(&self) -> PathBuf {
-        self.workdir.join("tenants")
+        self.workdir.join(TENANTS_SEGMENT_NAME)
     }
 
     pub fn tenant_path(&self, tenant_id: &TenantId) -> PathBuf {
diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs
index 3256a00182..02df8af51c 100644
--- a/pageserver/src/tenant.rs
+++ b/pageserver/src/tenant.rs
@@ -141,6 +141,9 @@ pub use crate::tenant::metadata::save_metadata;
 // re-export for use in walreceiver
 pub use crate::tenant::timeline::WalReceiverInfo;
 
+/// The "tenants" part of `tenants/<tenant>/timelines...`
+pub const TENANTS_SEGMENT_NAME: &str = "tenants";
+
 /// Parts of the `.neon/tenants/<tenant_id>/timelines/<timeline_id>` directory prefix.
 pub const TIMELINES_SEGMENT_NAME: &str = "timelines";
 
diff --git a/pageserver/src/virtual_file.rs b/pageserver/src/virtual_file.rs
index 7e9b50dfeb..91430ef6eb 100644
--- a/pageserver/src/virtual_file.rs
+++ b/pageserver/src/virtual_file.rs
@@ -11,6 +11,7 @@
 //! src/backend/storage/file/fd.c
 //!
 use crate::metrics::{STORAGE_IO_SIZE, STORAGE_IO_TIME};
+use crate::tenant::TENANTS_SEGMENT_NAME;
 use once_cell::sync::OnceCell;
 use std::fs::{self, File, OpenOptions};
 use std::io::{Error, ErrorKind, Seek, SeekFrom, Write};
@@ -235,7 +236,7 @@ impl VirtualFile {
         let parts = path_str.split('/').collect::<Vec<&str>>();
         let tenant_id;
         let timeline_id;
-        if parts.len() > 5 && parts[parts.len() - 5] == "tenants" {
+        if parts.len() > 5 && parts[parts.len() - 5] == TENANTS_SEGMENT_NAME {
             tenant_id = parts[parts.len() - 4].to_string();
             timeline_id = parts[parts.len() - 2].to_string();
         } else {
diff --git a/s3_scrubber/Cargo.toml b/s3_scrubber/Cargo.toml
index 47668eb4aa..82cd105d4f 100644
--- a/s3_scrubber/Cargo.toml
+++ b/s3_scrubber/Cargo.toml
@@ -22,6 +22,10 @@ serde_json.workspace = true
 serde_with.workspace = true
 workspace_hack.workspace = true
 utils.workspace = true
+async-stream.workspace = true
+tokio-stream.workspace = true
+futures-util.workspace = true
+itertools.workspace = true
 
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
 chrono = { workspace = true, default-features = false, features = ["clock", "serde"] }
@@ -30,10 +34,8 @@ aws-config = { workspace = true, default-features = false, features = ["rustls",
 
 pageserver = {path="../pageserver"}
 
-
 tracing.workspace = true
 tracing-subscriber.workspace = true
 clap.workspace = true
-
-atty = "0.2"
-tracing-appender = "0.2"
\ No newline at end of file
+tracing-appender = "0.2"
+histogram = "0.7"
\ No newline at end of file
diff --git a/s3_scrubber/src/checks.rs b/s3_scrubber/src/checks.rs
index c52a40ee94..e86c381985 100644
--- a/s3_scrubber/src/checks.rs
+++ b/s3_scrubber/src/checks.rs
@@ -4,13 +4,12 @@ use std::time::Duration;
 
 use anyhow::Context;
 use aws_sdk_s3::Client;
-use tokio::io::AsyncReadExt;
 use tokio::task::JoinSet;
 use tracing::{error, info, info_span, warn, Instrument};
 
 use crate::cloud_admin_api::{BranchData, CloudAdminApiClient, ProjectId};
 use crate::delete_batch_producer::DeleteProducerStats;
-use crate::{list_objects_with_retries, RootTarget, MAX_RETRIES};
+use crate::{download_object_with_retries, list_objects_with_retries, RootTarget, MAX_RETRIES};
 use pageserver::tenant::storage_layer::LayerFileName;
 use pageserver::tenant::IndexPart;
 use utils::id::TenantTimelineId;
@@ -92,9 +91,9 @@ pub async fn validate_pageserver_active_tenant_and_timelines(
             branch_checks.spawn(
                 async move {
                     let check_errors = branch_cleanup_and_check_errors(
-                        id,
+                        &id,
                         &s3_root,
-                        &s3_active_branch,
+                        Some(&s3_active_branch),
                         console_branch,
                         s3_data,
                     )
@@ -107,13 +106,13 @@ pub async fn validate_pageserver_active_tenant_and_timelines(
     }
 
     let mut total_stats = BranchCheckStats::default();
-    while let Some((id, branch_check_errors)) = branch_checks
+    while let Some((id, analysis)) = branch_checks
         .join_next()
         .await
         .transpose()
         .context("branch check task join")?
     {
-        total_stats.add(id, branch_check_errors);
+        total_stats.add(id, analysis.errors);
     }
     Ok(total_stats)
 }
@@ -160,35 +159,59 @@ impl BranchCheckStats {
     }
 }
 
-async fn branch_cleanup_and_check_errors(
-    id: TenantTimelineId,
+pub struct TimelineAnalysis {
+    /// Anomalies detected
+    pub errors: Vec<String>,
+
+    /// Healthy-but-noteworthy, like old-versioned structures that are readable but
+    /// worth reporting for awareness that we must not remove that old version decoding
+    /// yet.
+    pub warnings: Vec<String>,
+
+    /// Keys not referenced in metadata: candidates for removal
+    pub garbage_keys: Vec<String>,
+}
+
+impl TimelineAnalysis {
+    fn new() -> Self {
+        Self {
+            errors: Vec::new(),
+            warnings: Vec::new(),
+            garbage_keys: Vec::new(),
+        }
+    }
+}
+
+pub async fn branch_cleanup_and_check_errors(
+    id: &TenantTimelineId,
     s3_root: &RootTarget,
-    s3_active_branch: &BranchData,
+    s3_active_branch: Option<&BranchData>,
     console_branch: Option<BranchData>,
     s3_data: Option<S3TimelineBlobData>,
-) -> Vec<String> {
-    info!(
-        "Checking timeline for branch branch {:?}/{:?}",
-        s3_active_branch.project_id, s3_active_branch.id
-    );
-    let mut branch_check_errors = Vec::new();
+) -> TimelineAnalysis {
+    let mut result = TimelineAnalysis::new();
 
-    match console_branch {
-        Some(console_active_branch) => {
-            if console_active_branch.deleted {
-                branch_check_errors.push(format!("Timeline has deleted branch data in the console (id = {:?}, project_id = {:?}), recheck whether if it got removed during the check",
-                    s3_active_branch.id, s3_active_branch.project_id))
-            }
-        },
-        None => branch_check_errors.push(format!("Timeline has no branch data in the console (id = {:?}, project_id = {:?}), recheck whether if it got removed during the check",
+    info!("Checking timeline {id}");
+
+    if let Some(s3_active_branch) = s3_active_branch {
+        info!(
+            "Checking console status for timeline for branch {:?}/{:?}",
+            s3_active_branch.project_id, s3_active_branch.id
+        );
+        match console_branch {
+            Some(_) => {result.errors.push(format!("Timeline has deleted branch data in the console (id = {:?}, project_id = {:?}), recheck whether it got removed during the check",
+                s3_active_branch.id, s3_active_branch.project_id))
+            },
+            None => {
+                result.errors.push(format!("Timeline has no branch data in the console (id = {:?}, project_id = {:?}), recheck whether it got removed during the check",
             s3_active_branch.id, s3_active_branch.project_id))
+            }
+        };
     }
 
-    let mut keys_to_remove = Vec::new();
-
     match s3_data {
         Some(s3_data) => {
-            keys_to_remove.extend(s3_data.keys_to_remove);
+            result.garbage_keys.extend(s3_data.keys_to_remove);
 
             match s3_data.blob_data {
                 BlobDataParseResult::Parsed {
@@ -196,16 +219,23 @@ async fn branch_cleanup_and_check_errors(
                     mut s3_layers,
                 } => {
                     if !IndexPart::KNOWN_VERSIONS.contains(&index_part.get_version()) {
-                        branch_check_errors.push(format!(
+                        result.errors.push(format!(
                             "index_part.json version: {}",
                             index_part.get_version()
                         ))
                     }
 
+                    if &index_part.get_version() != IndexPart::KNOWN_VERSIONS.last().unwrap() {
+                        result.warnings.push(format!(
+                            "index_part.json version is not latest: {}",
+                            index_part.get_version()
+                        ))
+                    }
+
                     if index_part.metadata.disk_consistent_lsn()
                         != index_part.get_disk_consistent_lsn()
                     {
-                        branch_check_errors.push(format!(
+                        result.errors.push(format!(
                                     "Mismatching disk_consistent_lsn in TimelineMetadata ({}) and in the index_part ({})",
                                     index_part.metadata.disk_consistent_lsn(),
                                     index_part.get_disk_consistent_lsn(),
@@ -220,13 +250,13 @@ async fn branch_cleanup_and_check_errors(
 
                     for (layer, metadata) in index_part.layer_metadata {
                         if metadata.file_size == 0 {
-                            branch_check_errors.push(format!(
+                            result.errors.push(format!(
                                             "index_part.json contains a layer {} that has 0 size in its layer metadata", layer.file_name(),
                                         ))
                         }
 
                         if !s3_layers.remove(&layer) {
-                            branch_check_errors.push(format!(
+                            result.errors.push(format!(
                                 "index_part.json contains a layer {} that is not present in S3",
                                 layer.file_name(),
                             ))
@@ -234,55 +264,66 @@ async fn branch_cleanup_and_check_errors(
                     }
 
                     if !s3_layers.is_empty() {
-                        branch_check_errors.push(format!(
+                        result.errors.push(format!(
                             "index_part.json does not contain layers from S3: {:?}",
                             s3_layers
                                 .iter()
                                 .map(|layer_name| layer_name.file_name())
                                 .collect::<Vec<_>>(),
                         ));
-                        keys_to_remove.extend(s3_layers.iter().map(|layer_name| {
-                            let mut key = s3_root.timeline_root(id).prefix_in_bucket;
-                            let delimiter = s3_root.delimiter();
-                            if !key.ends_with(delimiter) {
-                                key.push_str(delimiter);
-                            }
-                            key.push_str(&layer_name.file_name());
-                            key
-                        }));
+                        result
+                            .garbage_keys
+                            .extend(s3_layers.iter().map(|layer_name| {
+                                let mut key = s3_root.timeline_root(id).prefix_in_bucket;
+                                let delimiter = s3_root.delimiter();
+                                if !key.ends_with(delimiter) {
+                                    key.push_str(delimiter);
+                                }
+                                key.push_str(&layer_name.file_name());
+                                key
+                            }));
                     }
                 }
-                BlobDataParseResult::Incorrect(parse_errors) => branch_check_errors.extend(
+                BlobDataParseResult::Incorrect(parse_errors) => result.errors.extend(
                     parse_errors
                         .into_iter()
                         .map(|error| format!("parse error: {error}")),
                 ),
             }
         }
-        None => branch_check_errors.push("Timeline has no data on S3 at all".to_string()),
+        None => result
+            .errors
+            .push("Timeline has no data on S3 at all".to_string()),
     }
 
-    if branch_check_errors.is_empty() {
+    if result.errors.is_empty() {
         info!("No check errors found");
     } else {
-        warn!("Found check errors: {branch_check_errors:?}");
+        warn!("Timeline metadata errors: {0:?}", result.errors);
     }
 
-    if !keys_to_remove.is_empty() {
-        error!("The following keys should be removed from S3: {keys_to_remove:?}")
+    if !result.warnings.is_empty() {
+        warn!("Timeline metadata warnings: {0:?}", result.warnings);
     }
 
-    branch_check_errors
+    if !result.garbage_keys.is_empty() {
+        error!(
+            "The following keys should be removed from S3: {0:?}",
+            result.garbage_keys
+        )
+    }
+
+    result
 }
 
 #[derive(Debug)]
-struct S3TimelineBlobData {
-    blob_data: BlobDataParseResult,
-    keys_to_remove: Vec<String>,
+pub struct S3TimelineBlobData {
+    pub blob_data: BlobDataParseResult,
+    pub keys_to_remove: Vec<String>,
 }
 
 #[derive(Debug)]
-enum BlobDataParseResult {
+pub enum BlobDataParseResult {
     Parsed {
         index_part: IndexPart,
         s3_layers: HashSet<LayerFileName>,
@@ -290,7 +331,7 @@ enum BlobDataParseResult {
     Incorrect(Vec<String>),
 }
 
-async fn list_timeline_blobs(
+pub async fn list_timeline_blobs(
     s3_client: &Client,
     id: TenantTimelineId,
     s3_root: &RootTarget,
@@ -298,7 +339,7 @@ async fn list_timeline_blobs(
     let mut s3_layers = HashSet::new();
     let mut index_part_object = None;
 
-    let timeline_dir_target = s3_root.timeline_root(id);
+    let timeline_dir_target = s3_root.timeline_root(&id);
     let mut continuation_token = None;
 
     let mut errors = Vec::new();
@@ -394,45 +435,3 @@ async fn list_timeline_blobs(
         keys_to_remove,
     })
 }
-
-async fn download_object_with_retries(
-    s3_client: &Client,
-    bucket_name: &str,
-    key: &str,
-) -> anyhow::Result<Vec<u8>> {
-    for _ in 0..MAX_RETRIES {
-        let mut body_buf = Vec::new();
-        let response_stream = match s3_client
-            .get_object()
-            .bucket(bucket_name)
-            .key(key)
-            .send()
-            .await
-        {
-            Ok(response) => response,
-            Err(e) => {
-                error!("Failed to download object for key {key}: {e}");
-                tokio::time::sleep(Duration::from_secs(1)).await;
-                continue;
-            }
-        };
-
-        match response_stream
-            .body
-            .into_async_read()
-            .read_to_end(&mut body_buf)
-            .await
-        {
-            Ok(bytes_read) => {
-                info!("Downloaded {bytes_read} bytes for object object with key {key}");
-                return Ok(body_buf);
-            }
-            Err(e) => {
-                error!("Failed to stream object body for key {key}: {e}");
-                tokio::time::sleep(Duration::from_secs(1)).await;
-            }
-        }
-    }
-
-    anyhow::bail!("Failed to download objects with key {key} {MAX_RETRIES} times")
-}
diff --git a/s3_scrubber/src/delete_batch_producer/timeline_batch.rs b/s3_scrubber/src/delete_batch_producer/timeline_batch.rs
index 0a0b31ae87..2ad522d3fb 100644
--- a/s3_scrubber/src/delete_batch_producer/timeline_batch.rs
+++ b/s3_scrubber/src/delete_batch_producer/timeline_batch.rs
@@ -41,7 +41,7 @@ pub async fn schedule_cleanup_deleted_timelines(
 
         let new_stats = async move {
             let tenant_id_to_check = project_to_check.tenant;
-            let check_target = check_root.timelines_root(tenant_id_to_check);
+            let check_target = check_root.timelines_root(&tenant_id_to_check);
             let stats = super::process_s3_target_recursively(
                 &check_s3_client,
                 &check_target,
diff --git a/s3_scrubber/src/lib.rs b/s3_scrubber/src/lib.rs
index ea1338cf11..f19a55efac 100644
--- a/s3_scrubber/src/lib.rs
+++ b/s3_scrubber/src/lib.rs
@@ -1,12 +1,15 @@
 pub mod checks;
 pub mod cloud_admin_api;
 pub mod delete_batch_producer;
+pub mod metadata_stream;
 mod s3_deletion;
+pub mod scan_metadata;
 
 use std::env;
 use std::fmt::Display;
 use std::time::Duration;
 
+use anyhow::Context;
 use aws_config::environment::EnvironmentVariableCredentialsProvider;
 use aws_config::imds::credentials::ImdsCredentialsProvider;
 use aws_config::meta::credentials::CredentialsProviderChain;
@@ -14,7 +17,9 @@ use aws_config::sso::SsoCredentialsProvider;
 use aws_sdk_s3::config::Region;
 use aws_sdk_s3::{Client, Config};
 
+use reqwest::Url;
 pub use s3_deletion::S3Deleter;
+use tokio::io::AsyncReadExt;
 use tracing::error;
 use tracing_appender::non_blocking::WorkerGuard;
 use tracing_subscriber::{fmt, prelude::*, EnvFilter};
@@ -23,6 +28,8 @@ use utils::id::{TenantId, TenantTimelineId};
 const MAX_RETRIES: usize = 20;
 const CLOUD_ADMIN_API_TOKEN_ENV_VAR: &str = "CLOUD_ADMIN_API_TOKEN";
 
+pub const CLI_NAME: &str = "s3-scrubber";
+
 #[derive(Debug, Clone)]
 pub struct S3Target {
     pub bucket_name: String,
@@ -69,19 +76,19 @@ impl RootTarget {
         }
     }
 
-    pub fn tenant_root(&self, tenant_id: TenantId) -> S3Target {
+    pub fn tenant_root(&self, tenant_id: &TenantId) -> S3Target {
         self.tenants_root().with_sub_segment(&tenant_id.to_string())
     }
 
-    pub fn timelines_root(&self, tenant_id: TenantId) -> S3Target {
+    pub fn timelines_root(&self, tenant_id: &TenantId) -> S3Target {
         match self {
             Self::Pageserver(_) => self.tenant_root(tenant_id).with_sub_segment("timelines"),
             Self::Safekeeper(_) => self.tenant_root(tenant_id),
         }
     }
 
-    pub fn timeline_root(&self, id: TenantTimelineId) -> S3Target {
-        self.timelines_root(id.tenant_id)
+    pub fn timeline_root(&self, id: &TenantTimelineId) -> S3Target {
+        self.timelines_root(&id.tenant_id)
             .with_sub_segment(&id.timeline_id.to_string())
     }
 
@@ -100,6 +107,55 @@ impl RootTarget {
     }
 }
 
+pub struct BucketConfig {
+    pub region: String,
+    pub bucket: String,
+
+    /// Use SSO if this is set, else rely on AWS_* environment vars
+    pub sso_account_id: Option<String>,
+}
+
+impl Display for BucketConfig {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{}/{}/{}",
+            self.sso_account_id.as_deref().unwrap_or("<none>"),
+            self.region,
+            self.bucket
+        )
+    }
+}
+
+impl BucketConfig {
+    pub fn from_env() -> anyhow::Result<Self> {
+        let sso_account_id = env::var("SSO_ACCOUNT_ID").ok();
+        let region = env::var("REGION").context("'REGION' param retrieval")?;
+        let bucket = env::var("BUCKET").context("'BUCKET' param retrieval")?;
+
+        Ok(Self {
+            region,
+            bucket,
+            sso_account_id,
+        })
+    }
+}
+
+pub struct ConsoleConfig {
+    pub admin_api_url: Url,
+}
+
+impl ConsoleConfig {
+    pub fn from_env() -> anyhow::Result<Self> {
+        let admin_api_url: Url = env::var("CLOUD_ADMIN_API_URL")
+            .context("'CLOUD_ADMIN_API_URL' param retrieval")?
+            .parse()
+            .context("'CLOUD_ADMIN_API_URL' param parsing")?;
+
+        Ok(Self { admin_api_url })
+    }
+}
+
 pub fn get_cloud_admin_api_token_or_exit() -> String {
     match env::var(CLOUD_ADMIN_API_TOKEN_ENV_VAR) {
         Ok(token) => token,
@@ -114,23 +170,7 @@ pub fn get_cloud_admin_api_token_or_exit() -> String {
     }
 }
 
-pub fn init_logging(binary_name: &str, dry_run: bool, node_kind: &str) -> WorkerGuard {
-    let file_name = if dry_run {
-        format!(
-            "{}_{}_{}__dry.log",
-            binary_name,
-            node_kind,
-            chrono::Utc::now().format("%Y_%m_%d__%H_%M_%S")
-        )
-    } else {
-        format!(
-            "{}_{}_{}.log",
-            binary_name,
-            node_kind,
-            chrono::Utc::now().format("%Y_%m_%d__%H_%M_%S")
-        )
-    };
-
+pub fn init_logging(file_name: &str) -> WorkerGuard {
     let (file_writer, guard) =
         tracing_appender::non_blocking(tracing_appender::rolling::never("./logs/", file_name));
 
@@ -140,7 +180,6 @@ pub fn init_logging(binary_name: &str, dry_run: bool, node_kind: &str) -> Worker
         .with_writer(file_writer);
     let stdout_logs = fmt::Layer::new()
         .with_target(false)
-        .with_ansi(atty::is(atty::Stream::Stdout))
         .with_writer(std::io::stdout);
     tracing_subscriber::registry()
         .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")))
@@ -151,30 +190,43 @@ pub fn init_logging(binary_name: &str, dry_run: bool, node_kind: &str) -> Worker
     guard
 }
 
-pub fn init_s3_client(account_id: String, bucket_region: Region) -> Client {
+pub fn init_s3_client(account_id: Option<String>, bucket_region: Region) -> Client {
     let credentials_provider = {
         // uses "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"
-        CredentialsProviderChain::first_try("env", EnvironmentVariableCredentialsProvider::new())
-            // uses sso
-            .or_else(
+        let chain = CredentialsProviderChain::first_try(
+            "env",
+            EnvironmentVariableCredentialsProvider::new(),
+        );
+
+        // Use SSO if we were given an account ID
+        match account_id {
+            Some(sso_account) => chain.or_else(
                 "sso",
                 SsoCredentialsProvider::builder()
-                    .account_id(account_id)
+                    .account_id(sso_account)
                     .role_name("PowerUserAccess")
                     .start_url("https://neondb.awsapps.com/start")
                     .region(Region::from_static("eu-central-1"))
                     .build(),
-            )
-            // uses imds v2
-            .or_else("imds", ImdsCredentialsProvider::builder().build())
+            ),
+            None => chain,
+        }
+        .or_else(
+            // Finally try IMDS
+            "imds",
+            ImdsCredentialsProvider::builder().build(),
+        )
     };
 
-    let config = Config::builder()
+    let mut builder = Config::builder()
         .region(bucket_region)
-        .credentials_provider(credentials_provider)
-        .build();
+        .credentials_provider(credentials_provider);
 
-    Client::from_conf(config)
+    if let Ok(endpoint) = env::var("AWS_ENDPOINT_URL") {
+        builder = builder.endpoint_url(endpoint)
+    }
+
+    Client::from_conf(builder.build())
 }
 
 async fn list_objects_with_retries(
@@ -202,3 +254,45 @@ async fn list_objects_with_retries(
 
     anyhow::bail!("Failed to list objects {MAX_RETRIES} times")
 }
+
+async fn download_object_with_retries(
+    s3_client: &Client,
+    bucket_name: &str,
+    key: &str,
+) -> anyhow::Result<Vec<u8>> {
+    for _ in 0..MAX_RETRIES {
+        let mut body_buf = Vec::new();
+        let response_stream = match s3_client
+            .get_object()
+            .bucket(bucket_name)
+            .key(key)
+            .send()
+            .await
+        {
+            Ok(response) => response,
+            Err(e) => {
+                error!("Failed to download object for key {key}: {e}");
+                tokio::time::sleep(Duration::from_secs(1)).await;
+                continue;
+            }
+        };
+
+        match response_stream
+            .body
+            .into_async_read()
+            .read_to_end(&mut body_buf)
+            .await
+        {
+            Ok(bytes_read) => {
+                tracing::info!("Downloaded {bytes_read} bytes for object object with key {key}");
+                return Ok(body_buf);
+            }
+            Err(e) => {
+                error!("Failed to stream object body for key {key}: {e}");
+                tokio::time::sleep(Duration::from_secs(1)).await;
+            }
+        }
+    }
+
+    anyhow::bail!("Failed to download objects with key {key} {MAX_RETRIES} times")
+}
diff --git a/s3_scrubber/src/main.rs b/s3_scrubber/src/main.rs
index 7004bcad51..3c60723f88 100644
--- a/s3_scrubber/src/main.rs
+++ b/s3_scrubber/src/main.rs
@@ -1,19 +1,18 @@
 use std::collections::HashMap;
-use std::env;
 use std::fmt::Display;
 use std::num::NonZeroUsize;
 use std::sync::Arc;
 
 use anyhow::Context;
 use aws_sdk_s3::config::Region;
-use reqwest::Url;
 use s3_scrubber::cloud_admin_api::CloudAdminApiClient;
 use s3_scrubber::delete_batch_producer::DeleteBatchProducer;
+use s3_scrubber::scan_metadata::scan_metadata;
 use s3_scrubber::{
-    checks, get_cloud_admin_api_token_or_exit, init_logging, init_s3_client, RootTarget, S3Deleter,
-    S3Target, TraversingDepth,
+    checks, get_cloud_admin_api_token_or_exit, init_logging, init_s3_client, BucketConfig,
+    ConsoleConfig, RootTarget, S3Deleter, S3Target, TraversingDepth, CLI_NAME,
 };
-use tracing::{info, info_span, warn};
+use tracing::{info, warn};
 
 use clap::{Parser, Subcommand, ValueEnum};
 
@@ -59,48 +58,7 @@ enum Command {
         #[arg(short, long, default_value_t = false)]
         skip_validation: bool,
     },
-}
-
-struct BucketConfig {
-    region: String,
-    bucket: String,
-    sso_account_id: String,
-}
-
-impl Display for BucketConfig {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}/{}/{}", self.sso_account_id, self.region, self.bucket)
-    }
-}
-
-impl BucketConfig {
-    fn from_env() -> anyhow::Result<Self> {
-        let sso_account_id =
-            env::var("SSO_ACCOUNT_ID").context("'SSO_ACCOUNT_ID' param retrieval")?;
-        let region = env::var("REGION").context("'REGION' param retrieval")?;
-        let bucket = env::var("BUCKET").context("'BUCKET' param retrieval")?;
-
-        Ok(Self {
-            region,
-            bucket,
-            sso_account_id,
-        })
-    }
-}
-
-struct ConsoleConfig {
-    admin_api_url: Url,
-}
-
-impl ConsoleConfig {
-    fn from_env() -> anyhow::Result<Self> {
-        let admin_api_url: Url = env::var("CLOUD_ADMIN_API_URL")
-            .context("'CLOUD_ADMIN_API_URL' param retrieval")?
-            .parse()
-            .context("'CLOUD_ADMIN_API_URL' param parsing")?;
-
-        Ok(Self { admin_api_url })
-    }
+    ScanMetadata {},
 }
 
 async fn tidy(
@@ -111,13 +69,24 @@ async fn tidy(
     depth: TraversingDepth,
     skip_validation: bool,
 ) -> anyhow::Result<()> {
-    let binary_name = env::args()
-        .next()
-        .context("binary name in not the first argument")?;
-
     let dry_run = !cli.delete;
-    let _guard = init_logging(&binary_name, dry_run, node_kind.as_str());
-    let _main_span = info_span!("tidy", binary = %binary_name, %dry_run).entered();
+    let file_name = if dry_run {
+        format!(
+            "{}_{}_{}__dry.log",
+            CLI_NAME,
+            node_kind,
+            chrono::Utc::now().format("%Y_%m_%d__%H_%M_%S")
+        )
+    } else {
+        format!(
+            "{}_{}_{}.log",
+            CLI_NAME,
+            node_kind,
+            chrono::Utc::now().format("%Y_%m_%d__%H_%M_%S")
+        )
+    };
+
+    let _guard = init_logging(&file_name);
 
     if dry_run {
         info!("Dry run, not removing items for real");
@@ -247,7 +216,7 @@ async fn main() -> anyhow::Result<()> {
 
     let bucket_config = BucketConfig::from_env()?;
 
-    match &cli.command {
+    match cli.command {
         Command::Tidy {
             node_kind,
             depth,
@@ -258,11 +227,25 @@ async fn main() -> anyhow::Result<()> {
                 &cli,
                 bucket_config,
                 console_config,
-                *node_kind,
-                *depth,
-                *skip_validation,
+                node_kind,
+                depth,
+                skip_validation,
             )
             .await
         }
+        Command::ScanMetadata {} => match scan_metadata(bucket_config).await {
+            Err(e) => {
+                tracing::error!("Failed: {e}");
+                Err(e)
+            }
+            Ok(summary) => {
+                println!("{}", summary.summary_string());
+                if summary.is_fatal() {
+                    Err(anyhow::anyhow!("Fatal scrub errors detected"))
+                } else {
+                    Ok(())
+                }
+            }
+        },
     }
 }
diff --git a/s3_scrubber/src/metadata_stream.rs b/s3_scrubber/src/metadata_stream.rs
new file mode 100644
index 0000000000..4e500a96cf
--- /dev/null
+++ b/s3_scrubber/src/metadata_stream.rs
@@ -0,0 +1,106 @@
+use anyhow::Context;
+use async_stream::{stream, try_stream};
+use aws_sdk_s3::Client;
+use tokio_stream::Stream;
+
+use crate::{list_objects_with_retries, RootTarget, TenantId};
+use utils::id::{TenantTimelineId, TimelineId};
+
+/// Given an S3 bucket, output a stream of TenantIds discovered via ListObjectsv2
+pub fn stream_tenants<'a>(
+    s3_client: &'a Client,
+    target: &'a RootTarget,
+) -> impl Stream<Item = anyhow::Result<TenantId>> + 'a {
+    try_stream! {
+        let mut continuation_token = None;
+        loop {
+            let tenants_target = target.tenants_root();
+            let fetch_response =
+                list_objects_with_retries(s3_client, tenants_target, continuation_token.clone()).await?;
+
+            let new_entry_ids = fetch_response
+                .common_prefixes()
+                .unwrap_or_default()
+                .iter()
+                .filter_map(|prefix| prefix.prefix())
+                .filter_map(|prefix| -> Option<&str> {
+                    prefix
+                        .strip_prefix(&tenants_target.prefix_in_bucket)?
+                        .strip_suffix('/')
+                }).map(|entry_id_str| {
+                entry_id_str
+                    .parse()
+                    .with_context(|| format!("Incorrect entry id str: {entry_id_str}"))
+            });
+
+            for i in new_entry_ids {
+                yield i?;
+            }
+
+            match fetch_response.next_continuation_token {
+                Some(new_token) => continuation_token = Some(new_token),
+                None => break,
+            }
+        }
+    }
+}
+
+/// Given a TenantId, output a stream of the timelines within that tenant, discovered
+/// using ListObjectsv2.  The listing is done before the stream is built, so that this
+/// function can be used to generate concurrency on a stream using buffer_unordered.
+pub async fn stream_tenant_timelines<'a>(
+    s3_client: &'a Client,
+    target: &'a RootTarget,
+    tenant: TenantId,
+) -> anyhow::Result<impl Stream<Item = Result<TenantTimelineId, anyhow::Error>> + 'a> {
+    let mut timeline_ids: Vec<Result<TimelineId, anyhow::Error>> = Vec::new();
+    let mut continuation_token = None;
+    let timelines_target = target.timelines_root(&tenant);
+
+    loop {
+        tracing::info!("Listing in {}", tenant);
+        let fetch_response =
+            list_objects_with_retries(s3_client, &timelines_target, continuation_token.clone())
+                .await;
+        let fetch_response = match fetch_response {
+            Err(e) => {
+                timeline_ids.push(Err(e));
+                break;
+            }
+            Ok(r) => r,
+        };
+
+        let new_entry_ids = fetch_response
+            .common_prefixes()
+            .unwrap_or_default()
+            .iter()
+            .filter_map(|prefix| prefix.prefix())
+            .filter_map(|prefix| -> Option<&str> {
+                prefix
+                    .strip_prefix(&timelines_target.prefix_in_bucket)?
+                    .strip_suffix('/')
+            })
+            .map(|entry_id_str| {
+                entry_id_str
+                    .parse::<TimelineId>()
+                    .with_context(|| format!("Incorrect entry id str: {entry_id_str}"))
+            });
+
+        for i in new_entry_ids {
+            timeline_ids.push(i);
+        }
+
+        match fetch_response.next_continuation_token {
+            Some(new_token) => continuation_token = Some(new_token),
+            None => break,
+        }
+    }
+
+    tracing::info!("Yielding for {}", tenant);
+    Ok(stream! {
+        for i in timeline_ids {
+            let id = i?;
+            yield Ok(TenantTimelineId::new(tenant, id));
+        }
+    })
+}
diff --git a/s3_scrubber/src/s3_deletion.rs b/s3_scrubber/src/s3_deletion.rs
index 716443790b..a03cc65c89 100644
--- a/s3_scrubber/src/s3_deletion.rs
+++ b/s3_scrubber/src/s3_deletion.rs
@@ -164,7 +164,7 @@ async fn delete_tenants_batch(
         s3_target,
         s3_client,
         dry_run,
-        |root_target, tenant_to_delete| root_target.tenant_root(tenant_to_delete),
+        |root_target, tenant_to_delete| root_target.tenant_root(&tenant_to_delete),
     )
     .await?;
 
@@ -215,7 +215,7 @@ async fn delete_timelines_batch(
         s3_target,
         s3_client,
         dry_run,
-        |root_target, timeline_to_delete| root_target.timeline_root(timeline_to_delete),
+        |root_target, timeline_to_delete| root_target.timeline_root(&timeline_to_delete),
     )
     .await?;
 
@@ -386,7 +386,7 @@ async fn ensure_tenant_batch_deleted(
 
     for &tenant_id in batch {
         let fetch_response =
-            list_objects_with_retries(s3_client, &s3_target.tenant_root(tenant_id), None).await?;
+            list_objects_with_retries(s3_client, &s3_target.tenant_root(&tenant_id), None).await?;
 
         if fetch_response.is_truncated()
             || fetch_response.contents().is_some()
@@ -415,7 +415,7 @@ async fn ensure_timeline_batch_deleted(
 
     for &id in batch {
         let fetch_response =
-            list_objects_with_retries(s3_client, &s3_target.timeline_root(id), None).await?;
+            list_objects_with_retries(s3_client, &s3_target.timeline_root(&id), None).await?;
 
         if fetch_response.is_truncated()
             || fetch_response.contents().is_some()
diff --git a/s3_scrubber/src/scan_metadata.rs b/s3_scrubber/src/scan_metadata.rs
new file mode 100644
index 0000000000..f75d7645a8
--- /dev/null
+++ b/s3_scrubber/src/scan_metadata.rs
@@ -0,0 +1,234 @@
+use std::collections::{HashMap, HashSet};
+use std::sync::Arc;
+
+use crate::checks::{
+    branch_cleanup_and_check_errors, list_timeline_blobs, BlobDataParseResult, S3TimelineBlobData,
+    TimelineAnalysis,
+};
+use crate::metadata_stream::{stream_tenant_timelines, stream_tenants};
+use crate::{init_logging, init_s3_client, BucketConfig, RootTarget, S3Target, CLI_NAME};
+use aws_sdk_s3::Client;
+use aws_types::region::Region;
+use futures_util::{pin_mut, StreamExt, TryStreamExt};
+use histogram::Histogram;
+use pageserver::tenant::{IndexPart, TENANTS_SEGMENT_NAME};
+use utils::id::TenantTimelineId;
+
+pub struct MetadataSummary {
+    count: usize,
+    with_errors: HashSet<TenantTimelineId>,
+    with_warnings: HashSet<TenantTimelineId>,
+    with_garbage: HashSet<TenantTimelineId>,
+    indices_by_version: HashMap<usize, usize>,
+
+    layer_count: MinMaxHisto,
+    timeline_size_bytes: MinMaxHisto,
+    layer_size_bytes: MinMaxHisto,
+}
+
+/// A histogram plus minimum and maximum tracking
+struct MinMaxHisto {
+    histo: Histogram,
+    min: u64,
+    max: u64,
+}
+
+impl MinMaxHisto {
+    fn new() -> Self {
+        Self {
+            histo: histogram::Histogram::builder()
+                .build()
+                .expect("Bad histogram params"),
+            min: u64::MAX,
+            max: 0,
+        }
+    }
+
+    fn sample(&mut self, v: u64) -> Result<(), histogram::Error> {
+        self.min = std::cmp::min(self.min, v);
+        self.max = std::cmp::max(self.max, v);
+        let r = self.histo.increment(v, 1);
+
+        if r.is_err() {
+            tracing::warn!("Bad histogram sample: {v}");
+        }
+
+        r
+    }
+
+    fn oneline(&self) -> String {
+        let percentiles = match self.histo.percentiles(&[1.0, 10.0, 50.0, 90.0, 99.0]) {
+            Ok(p) => p,
+            Err(e) => return format!("No data: {}", e),
+        };
+
+        let percentiles: Vec<u64> = percentiles
+            .iter()
+            .map(|p| p.bucket().low() + p.bucket().high() / 2)
+            .collect();
+
+        format!(
+            "min {}, 1% {}, 10% {}, 50% {}, 90% {}, 99% {}, max {}",
+            self.min,
+            percentiles[0],
+            percentiles[1],
+            percentiles[2],
+            percentiles[3],
+            percentiles[4],
+            self.max,
+        )
+    }
+}
+
+impl MetadataSummary {
+    fn new() -> Self {
+        Self {
+            count: 0,
+            with_errors: HashSet::new(),
+            with_warnings: HashSet::new(),
+            with_garbage: HashSet::new(),
+            indices_by_version: HashMap::new(),
+            layer_count: MinMaxHisto::new(),
+            timeline_size_bytes: MinMaxHisto::new(),
+            layer_size_bytes: MinMaxHisto::new(),
+        }
+    }
+
+    fn update_histograms(&mut self, index_part: &IndexPart) -> Result<(), histogram::Error> {
+        self.layer_count
+            .sample(index_part.layer_metadata.len() as u64)?;
+        let mut total_size: u64 = 0;
+        for meta in index_part.layer_metadata.values() {
+            total_size += meta.file_size;
+            self.layer_size_bytes.sample(meta.file_size)?;
+        }
+        self.timeline_size_bytes.sample(total_size)?;
+
+        Ok(())
+    }
+
+    fn update_data(&mut self, data: &S3TimelineBlobData) {
+        self.count += 1;
+        if let BlobDataParseResult::Parsed {
+            index_part,
+            s3_layers: _,
+        } = &data.blob_data
+        {
+            *self
+                .indices_by_version
+                .entry(index_part.get_version())
+                .or_insert(0) += 1;
+
+            if let Err(e) = self.update_histograms(index_part) {
+                // Value out of range?  Warn that the results are untrustworthy
+                tracing::warn!(
+                    "Error updating histograms, summary stats may be wrong: {}",
+                    e
+                );
+            }
+        }
+    }
+
+    fn update_analysis(&mut self, id: &TenantTimelineId, analysis: &TimelineAnalysis) {
+        if !analysis.errors.is_empty() {
+            self.with_errors.insert(*id);
+        }
+
+        if !analysis.warnings.is_empty() {
+            self.with_warnings.insert(*id);
+        }
+    }
+
+    /// Long-form output for printing at end of a scan
+    pub fn summary_string(&self) -> String {
+        let version_summary: String = itertools::join(
+            self.indices_by_version
+                .iter()
+                .map(|(k, v)| format!("{k}: {v}")),
+            ", ",
+        );
+
+        format!(
+            "Timelines: {0}
+With errors: {1}
+With warnings: {2}
+With garbage: {3}
+Index versions: {version_summary}
+Timeline size bytes: {4}
+Layer size bytes: {5}
+Timeline layer count: {6}
+",
+            self.count,
+            self.with_errors.len(),
+            self.with_warnings.len(),
+            self.with_garbage.len(),
+            self.timeline_size_bytes.oneline(),
+            self.layer_size_bytes.oneline(),
+            self.layer_count.oneline(),
+        )
+    }
+
+    pub fn is_fatal(&self) -> bool {
+        !self.with_errors.is_empty()
+    }
+}
+
+/// Scan the pageserver metadata in an S3 bucket, reporting errors and statistics.
+pub async fn scan_metadata(bucket_config: BucketConfig) -> anyhow::Result<MetadataSummary> {
+    let file_name = format!(
+        "{}_scan_metadata_{}_{}.log",
+        CLI_NAME,
+        bucket_config.bucket,
+        chrono::Utc::now().format("%Y_%m_%d__%H_%M_%S")
+    );
+
+    let _guard = init_logging(&file_name);
+
+    let s3_client = Arc::new(init_s3_client(
+        bucket_config.sso_account_id,
+        Region::new(bucket_config.region),
+    ));
+    let delimiter = "/";
+    let target = RootTarget::Pageserver(S3Target {
+        bucket_name: bucket_config.bucket.to_string(),
+        prefix_in_bucket: ["pageserver", "v1", TENANTS_SEGMENT_NAME, ""].join(delimiter),
+        delimiter: delimiter.to_string(),
+    });
+
+    let tenants = stream_tenants(&s3_client, &target);
+
+    // How many tenants to process in parallel.  We need to be mindful of pageservers
+    // accessing the same per tenant prefixes, so use a lower setting than pageservers.
+    const CONCURRENCY: usize = 32;
+
+    // Generate a stream of TenantTimelineId
+    let timelines = tenants.map_ok(|t| stream_tenant_timelines(&s3_client, &target, t));
+    let timelines = timelines.try_buffer_unordered(CONCURRENCY);
+    let timelines = timelines.try_flatten();
+
+    // Generate a stream of S3TimelineBlobData
+    async fn report_on_timeline(
+        s3_client: &Client,
+        target: &RootTarget,
+        ttid: TenantTimelineId,
+    ) -> anyhow::Result<(TenantTimelineId, S3TimelineBlobData)> {
+        let data = list_timeline_blobs(s3_client, ttid, target).await?;
+        Ok((ttid, data))
+    }
+    let timelines = timelines.map_ok(|ttid| report_on_timeline(&s3_client, &target, ttid));
+    let timelines = timelines.try_buffer_unordered(CONCURRENCY);
+
+    let mut summary = MetadataSummary::new();
+    pin_mut!(timelines);
+    while let Some(i) = timelines.next().await {
+        let (ttid, data) = i?;
+        summary.update_data(&data);
+
+        let analysis =
+            branch_cleanup_and_check_errors(&ttid, &target, None, None, Some(data)).await;
+
+        summary.update_analysis(&ttid, &analysis);
+    }
+
+    Ok(summary)
+}
diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py
index 3936736e56..4ea1d79705 100644
--- a/test_runner/fixtures/neon_fixtures.py
+++ b/test_runner/fixtures/neon_fixtures.py
@@ -415,6 +415,7 @@ class NeonEnvBuilder:
         pg_distrib_dir: Path,
         pg_version: PgVersion,
         test_name: str,
+        test_output_dir: Path,
         remote_storage: Optional[RemoteStorage] = None,
         remote_storage_users: RemoteStorageUsers = RemoteStorageUsers.PAGESERVER,
         pageserver_config_override: Optional[str] = None,
@@ -455,6 +456,8 @@ class NeonEnvBuilder:
         self.preserve_database_files = preserve_database_files
         self.initial_tenant = initial_tenant or TenantId.generate()
         self.initial_timeline = initial_timeline or TimelineId.generate()
+        self.scrub_on_exit = False
+        self.test_output_dir = test_output_dir
 
         assert test_name.startswith(
             "test_"
@@ -489,6 +492,23 @@ class NeonEnvBuilder:
 
         return env
 
+    def enable_scrub_on_exit(self):
+        """
+        Call this if you would like the fixture to automatically run
+        s3_scrubber at the end of the test, as a bidirectional test
+        that the scrubber is working properly, and that the code within
+        the test didn't produce any invalid remote state.
+        """
+
+        if not isinstance(self.remote_storage, S3Storage):
+            # The scrubber can't talk to e.g. LocalFS -- it needs
+            # an HTTP endpoint (mock is fine) to connect to.
+            raise RuntimeError(
+                "Cannot scrub with remote_storage={self.remote_storage}, require an S3 endpoint"
+            )
+
+        self.scrub_on_exit = True
+
     def enable_remote_storage(
         self,
         remote_storage_kind: RemoteStorageKind,
@@ -721,11 +741,20 @@ class NeonEnvBuilder:
             self.env.pageserver.stop(immediate=True)
 
             cleanup_error = None
+
+            if self.scrub_on_exit:
+                try:
+                    S3Scrubber(self.test_output_dir, self).scan_metadata()
+                except Exception as e:
+                    log.error(f"Error during remote storage scrub: {e}")
+                    cleanup_error = e
+
             try:
                 self.cleanup_remote_storage()
             except Exception as e:
                 log.error(f"Error during remote storage cleanup: {e}")
-                cleanup_error = e
+                if cleanup_error is not None:
+                    cleanup_error = e
 
             try:
                 self.cleanup_local_storage()
@@ -929,6 +958,7 @@ def _shared_simple_env(
     default_broker: NeonBroker,
     run_id: uuid.UUID,
     top_output_dir: Path,
+    test_output_dir: Path,
     neon_binpath: Path,
     pg_distrib_dir: Path,
     pg_version: PgVersion,
@@ -957,6 +987,7 @@ def _shared_simple_env(
         run_id=run_id,
         preserve_database_files=pytestconfig.getoption("--preserve-database-files"),
         test_name=request.node.name,
+        test_output_dir=test_output_dir,
     ) as builder:
         env = builder.init_start()
 
@@ -984,7 +1015,7 @@ def neon_simple_env(_shared_simple_env: NeonEnv) -> Iterator[NeonEnv]:
 @pytest.fixture(scope="function")
 def neon_env_builder(
     pytestconfig: Config,
-    test_output_dir: str,
+    test_output_dir: Path,
     port_distributor: PortDistributor,
     mock_s3_server: MockS3Server,
     neon_binpath: Path,
@@ -1022,6 +1053,7 @@ def neon_env_builder(
         run_id=run_id,
         preserve_database_files=pytestconfig.getoption("--preserve-database-files"),
         test_name=request.node.name,
+        test_output_dir=test_output_dir,
     ) as builder:
         yield builder
 
@@ -1728,7 +1760,10 @@ class PgBin:
         self._fixpath(command)
         log.info(f"Running command '{' '.join(command)}'")
         env = self._build_env(env)
-        return subprocess_capture(self.log_dir, command, env=env, cwd=cwd, check=True, **kwargs)
+        base_path, _, _ = subprocess_capture(
+            self.log_dir, command, env=env, cwd=cwd, check=True, **kwargs
+        )
+        return base_path
 
 
 @pytest.fixture(scope="function")
@@ -2734,6 +2769,41 @@ class SafekeeperHttpClient(requests.Session):
         return metrics
 
 
+class S3Scrubber:
+    def __init__(self, log_dir: Path, env: NeonEnvBuilder):
+        self.env = env
+        self.log_dir = log_dir
+
+    def scrubber_cli(self, args, timeout):
+        assert isinstance(self.env.remote_storage, S3Storage)
+        s3_storage = self.env.remote_storage
+
+        env = {
+            "REGION": s3_storage.bucket_region,
+            "BUCKET": s3_storage.bucket_name,
+        }
+        env.update(s3_storage.access_env_vars())
+
+        if s3_storage.endpoint is not None:
+            env.update({"AWS_ENDPOINT_URL": s3_storage.endpoint})
+
+        base_args = [self.env.neon_binpath / "s3_scrubber"]
+        args = base_args + args
+
+        (output_path, _, status_code) = subprocess_capture(
+            self.log_dir, args, echo_stderr=True, echo_stdout=True, env=env, check=False
+        )
+        if status_code:
+            log.warning(f"Scrub command {args} failed")
+            log.warning(f"Scrub environment: {env}")
+            log.warning(f"Output at: {output_path}")
+
+            raise RuntimeError("Remote storage scrub failed")
+
+    def scan_metadata(self):
+        self.scrubber_cli(["scan-metadata"], timeout=30)
+
+
 def get_test_output_dir(request: FixtureRequest, top_output_dir: Path) -> Path:
     """Compute the working directory for an individual test."""
     test_name = request.node.name
diff --git a/test_runner/fixtures/remote_storage.py b/test_runner/fixtures/remote_storage.py
index 320e658639..a68257bbac 100644
--- a/test_runner/fixtures/remote_storage.py
+++ b/test_runner/fixtures/remote_storage.py
@@ -88,6 +88,19 @@ def available_s3_storages() -> List[RemoteStorageKind]:
     return remote_storages
 
 
+def s3_storage() -> RemoteStorageKind:
+    """
+    For tests that require a remote storage impl that exposes an S3
+    endpoint, but don't want to parametrize over multiple storage types.
+
+    Use real S3 if available, else use MockS3
+    """
+    if os.getenv("ENABLE_REAL_S3_REMOTE_STORAGE") is not None:
+        return RemoteStorageKind.REAL_S3
+    else:
+        return RemoteStorageKind.MOCK_S3
+
+
 @dataclass
 class LocalFsStorage:
     root: Path
diff --git a/test_runner/fixtures/utils.py b/test_runner/fixtures/utils.py
index d03d2e7595..46ab446f99 100644
--- a/test_runner/fixtures/utils.py
+++ b/test_runner/fixtures/utils.py
@@ -4,9 +4,10 @@ import os
 import re
 import subprocess
 import tarfile
+import threading
 import time
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, Dict, List, Tuple, TypeVar
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, TypeVar
 from urllib.parse import urlencode
 
 import allure
@@ -26,34 +27,100 @@ def get_self_dir() -> Path:
     return Path(__file__).resolve().parent
 
 
-def subprocess_capture(capture_dir: Path, cmd: List[str], **kwargs: Any) -> str:
-    """Run a process and capture its output
+def subprocess_capture(
+    capture_dir: Path,
+    cmd: List[str],
+    *,
+    check=False,
+    echo_stderr=False,
+    echo_stdout=False,
+    capture_stdout=False,
+    **kwargs: Any,
+) -> Tuple[str, Optional[str], int]:
+    """Run a process and bifurcate its output to files and the `log` logger
 
-    Output will go to files named "cmd_NNN.stdout" and "cmd_NNN.stderr"
+    stderr and stdout are always captured in files.  They are also optionally
+    echoed to the log (echo_stderr, echo_stdout), and/or captured and returned
+    (capture_stdout).
+
+    File output will go to files named "cmd_NNN.stdout" and "cmd_NNN.stderr"
     where "cmd" is the name of the program and NNN is an incrementing
     counter.
 
     If those files already exist, we will overwrite them.
-    Returns basepath for files with captured output.
+
+    Returns 3-tuple of:
+     - The base path for output files
+     - Captured stdout, or None
+     - The exit status of the process
     """
     assert isinstance(cmd, list)
-    base = f"{os.path.basename(cmd[0])}_{global_counter()}"
+    base_cmd = os.path.basename(cmd[0])
+    base = f"{base_cmd}_{global_counter()}"
     basepath = os.path.join(capture_dir, base)
     stdout_filename = f"{basepath}.stdout"
     stderr_filename = f"{basepath}.stderr"
 
+    # Since we will stream stdout and stderr concurrently, need to do it in a thread.
+    class OutputHandler(threading.Thread):
+        def __init__(self, in_file, out_file, echo: bool, capture: bool):
+            super().__init__()
+            self.in_file = in_file
+            self.out_file = out_file
+            self.echo = echo
+            self.capture = capture
+            self.captured = ""
+
+        def run(self):
+            for line in self.in_file:
+                # Only bother decoding if we are going to do something more than stream to a file
+                if self.echo or self.capture:
+                    string = line.decode(encoding="utf-8", errors="replace")
+
+                    if self.echo:
+                        log.info(string)
+
+                    if self.capture:
+                        self.captured += string
+
+                self.out_file.write(line)
+
+    captured = None
     try:
-        with open(stdout_filename, "w") as stdout_f:
-            with open(stderr_filename, "w") as stderr_f:
+        with open(stdout_filename, "wb") as stdout_f:
+            with open(stderr_filename, "wb") as stderr_f:
                 log.info(f'Capturing stdout to "{base}.stdout" and stderr to "{base}.stderr"')
-                subprocess.run(cmd, **kwargs, stdout=stdout_f, stderr=stderr_f)
+
+                p = subprocess.Popen(
+                    cmd,
+                    **kwargs,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                )
+                stdout_handler = OutputHandler(
+                    p.stdout, stdout_f, echo=echo_stdout, capture=capture_stdout
+                )
+                stdout_handler.start()
+                stderr_handler = OutputHandler(p.stderr, stderr_f, echo=echo_stderr, capture=False)
+                stderr_handler.start()
+
+                r = p.wait()
+
+                stdout_handler.join()
+                stderr_handler.join()
+
+                if check and r != 0:
+                    raise subprocess.CalledProcessError(r, " ".join(cmd))
+
+                if capture_stdout:
+                    captured = stdout_handler.captured
     finally:
         # Remove empty files if there is no output
         for filename in (stdout_filename, stderr_filename):
             if os.stat(filename).st_size == 0:
                 os.remove(filename)
 
-    return basepath
+    return (basepath, captured, r)
 
 
 _global_counter = 0
diff --git a/test_runner/regress/test_pageserver_restart.py b/test_runner/regress/test_pageserver_restart.py
index 1e41ebd15b..567200ba1f 100644
--- a/test_runner/regress/test_pageserver_restart.py
+++ b/test_runner/regress/test_pageserver_restart.py
@@ -3,11 +3,15 @@ from contextlib import closing
 import pytest
 from fixtures.log_helper import log
 from fixtures.neon_fixtures import NeonEnvBuilder
+from fixtures.remote_storage import s3_storage
 
 
 # Test restarting page server, while safekeeper and compute node keep
 # running.
 def test_pageserver_restart(neon_env_builder: NeonEnvBuilder):
+    neon_env_builder.enable_remote_storage(remote_storage_kind=s3_storage())
+    neon_env_builder.enable_scrub_on_exit()
+
     env = neon_env_builder.init_start()
 
     env.neon_cli.create_branch("test_pageserver_restart")
@@ -109,6 +113,9 @@ def test_pageserver_restart(neon_env_builder: NeonEnvBuilder):
 # safekeeper and compute node keep running.
 @pytest.mark.timeout(540)
 def test_pageserver_chaos(neon_env_builder: NeonEnvBuilder):
+    neon_env_builder.enable_remote_storage(remote_storage_kind=s3_storage())
+    neon_env_builder.enable_scrub_on_exit()
+
     env = neon_env_builder.init_start()
 
     # Use a tiny checkpoint distance, to create a lot of layers quickly.

From da60f69909fc3b8ab7ea072b4196022917db3628 Mon Sep 17 00:00:00 2001
From: Alexander Bayandin <alexander@neon.tech>
Date: Wed, 6 Sep 2023 12:55:11 +0100
Subject: [PATCH 11/61] approved-for-ci-run.yml: use our bot (#5216)

## Problem

Pull Requests created by GitHub Actions bot doesn't have access to
secrets, so we need to use our bot for it to auto-trigger a tests run

See previous PRs  #4663, #5210, #5212

## Summary of changes
- Use our bot to create PRs
---
 .github/workflows/approved-for-ci-run.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/approved-for-ci-run.yml b/.github/workflows/approved-for-ci-run.yml
index 77cedae375..256ab8b438 100644
--- a/.github/workflows/approved-for-ci-run.yml
+++ b/.github/workflows/approved-for-ci-run.yml
@@ -17,7 +17,7 @@ on:
       - labeled
 
 env:
-  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
   PR_NUMBER: ${{ github.event.pull_request.number }}
 
 concurrency:
@@ -33,7 +33,6 @@ jobs:
       contains(github.event.pull_request.labels.*.name, 'approved-for-ci-run')
 
     runs-on: ubuntu-latest
-    permissions: write-all
 
     steps:
       - run: gh pr --repo "${GITHUB_REPOSITORY}" edit "${PR_NUMBER}" --remove-label "approved-for-ci-run"
@@ -46,7 +45,6 @@ jobs:
       contains(github.event.pull_request.labels.*.name, 'approved-for-ci-run')
 
     runs-on: ubuntu-latest
-    permissions: write-all
 
     steps:
       - run: gh pr --repo "${GITHUB_REPOSITORY}" edit "${PR_NUMBER}" --remove-label "approved-for-ci-run"

From 61d661a6c3da7697d02652c1c56133e53ed1bb1f Mon Sep 17 00:00:00 2001
From: John Spray <john@neon.tech>
Date: Wed, 6 Sep 2023 14:44:48 +0100
Subject: [PATCH 12/61] pageserver: generation number fetch on startup and use
 in /attach (#5163)

## Problem

- #5050

Closes: https://github.com/neondatabase/neon/issues/5136

## Summary of changes

- A new configuration property `control_plane_api` controls other
functionality in this PR: if it is unset (default) then everything still
works as it does today.
- If `control_plane_api` is set, then on startup we call out to control
plane `/re-attach` endpoint to discover our attachments and their
generations. If an attachment is missing from the response we implicitly
detach the tenant.
- Calls to pageserver `/attach` API may include a `generation`
parameter. If `control_plane_api` is set, then this parameter is
mandatory.
- RemoteTimelineClient's loading of index_part.json is generation-aware,
and will try to load the index_part with the most recent generation <=
its own generation.
- The `neon_local` testing environment now includes a new binary
`attachment_service` which implements the endpoints that the pageserver
requires to operate. This is on by default if running `cargo neon` by
hand. In `test_runner/` tests, it is off by default: existing tests
continue to run with in the legacy generation-less mode.

Caveats:
- The re-attachment during startup assumes that we are only re-attaching
tenants that have previously been attached, and not totally new tenants
-- this relies on the control plane's attachment logic to keep retrying
so that we should eventually see the attach API call. That's important
because the `/re-attach` API doesn't tell us which timelines we should
attach -- we still use local disk state for that. Ref:
https://github.com/neondatabase/neon/issues/5173
- Testing: generations are only enabled for one integration test right
now (test_pageserver_restart), as a smoke test that all the machinery
basically works. Writing fuller tests that stress tenant migration will
come later, and involve extending our test fixtures to deal with
multiple pageservers.
- I'm not in love with "attachment_service" as a name for the neon_local
component, but it's not very important because we can easily rename
these test bits whenever we want.
- Limited observability when in re-attach on startup: when I add
generation validation for deletions in a later PR, I want to wrap up the
control plane API calls in some small client class that will expose
metrics for things like errors calling the control plane API, which will
act as a strong red signal that something is not right.

Co-authored-by: Christian Schwarz <christian@neon.tech>
Co-authored-by: Joonas Koivunen <joonas@neon.tech>
---
 Cargo.lock                                    |   3 +
 control_plane/Cargo.toml                      |   3 +
 control_plane/src/attachment_service.rs       | 106 +++++++
 control_plane/src/bin/attachment_service.rs   | 273 ++++++++++++++++++
 control_plane/src/bin/neon_local.rs           |  86 +++++-
 control_plane/src/lib.rs                      |   1 +
 control_plane/src/local_env.rs                |   8 +
 control_plane/src/pageserver.rs               |  14 +-
 libs/pageserver_api/src/control_api.rs        |  52 ++++
 libs/pageserver_api/src/lib.rs                |   1 +
 libs/pageserver_api/src/models.rs             |  25 +-
 libs/remote_storage/src/local_fs.rs           |  46 ++-
 libs/utils/src/generation.rs                  |  25 ++
 pageserver/src/bin/pageserver.rs              |   1 +
 pageserver/src/config.rs                      |  17 ++
 pageserver/src/control_plane_client.rs        | 119 ++++++++
 pageserver/src/http/openapi_spec.yml          |   4 +-
 pageserver/src/http/routes.rs                 |  45 ++-
 pageserver/src/lib.rs                         |   1 +
 pageserver/src/tenant.rs                      |   2 +-
 pageserver/src/tenant/metadata.rs             |  17 ++
 pageserver/src/tenant/mgr.rs                  |  87 ++++--
 .../src/tenant/remote_timeline_client.rs      | 159 +++++++++-
 .../tenant/remote_timeline_client/download.rs | 110 ++++++-
 pageserver/src/tenant/timeline.rs             |  17 +-
 pageserver/src/tenant/timeline/init.rs        |   6 +-
 test_runner/fixtures/neon_fixtures.py         |  85 +++++-
 test_runner/fixtures/pageserver/http.py       |  15 +-
 .../regress/test_pageserver_restart.py        |   4 +-
 test_runner/regress/test_remote_storage.py    |  16 +-
 30 files changed, 1264 insertions(+), 84 deletions(-)
 create mode 100644 control_plane/src/attachment_service.rs
 create mode 100644 control_plane/src/bin/attachment_service.rs
 create mode 100644 libs/pageserver_api/src/control_api.rs
 create mode 100644 pageserver/src/control_plane_client.rs

diff --git a/Cargo.lock b/Cargo.lock
index 503c944aff..e811a06fac 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1057,6 +1057,8 @@ dependencies = [
  "comfy-table",
  "compute_api",
  "git-version",
+ "hex",
+ "hyper",
  "nix 0.26.2",
  "once_cell",
  "pageserver_api",
@@ -1072,6 +1074,7 @@ dependencies = [
  "storage_broker",
  "tar",
  "thiserror",
+ "tokio",
  "toml",
  "tracing",
  "url",
diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml
index d2c99c5f36..ec685915f9 100644
--- a/control_plane/Cargo.toml
+++ b/control_plane/Cargo.toml
@@ -12,6 +12,8 @@ git-version.workspace = true
 nix.workspace = true
 once_cell.workspace = true
 postgres.workspace = true
+hex.workspace = true
+hyper.workspace = true
 regex.workspace = true
 reqwest = { workspace = true, features = ["blocking", "json"] }
 serde.workspace = true
@@ -20,6 +22,7 @@ serde_with.workspace = true
 tar.workspace = true
 thiserror.workspace = true
 toml.workspace = true
+tokio.workspace = true
 url.workspace = true
 # Note: Do not directly depend on pageserver or safekeeper; use pageserver_api or safekeeper_api
 # instead, so that recompile times are better.
diff --git a/control_plane/src/attachment_service.rs b/control_plane/src/attachment_service.rs
new file mode 100644
index 0000000000..2bd4260aa8
--- /dev/null
+++ b/control_plane/src/attachment_service.rs
@@ -0,0 +1,106 @@
+use crate::{background_process, local_env::LocalEnv};
+use anyhow::anyhow;
+use serde::{Deserialize, Serialize};
+use serde_with::{serde_as, DisplayFromStr};
+use std::{path::PathBuf, process::Child};
+use utils::id::{NodeId, TenantId};
+
+pub struct AttachmentService {
+    env: LocalEnv,
+    listen: String,
+    path: PathBuf,
+}
+
+const COMMAND: &str = "attachment_service";
+
+#[serde_as]
+#[derive(Serialize, Deserialize)]
+pub struct AttachHookRequest {
+    #[serde_as(as = "DisplayFromStr")]
+    pub tenant_id: TenantId,
+    pub pageserver_id: Option<NodeId>,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct AttachHookResponse {
+    pub gen: Option<u32>,
+}
+
+impl AttachmentService {
+    pub fn from_env(env: &LocalEnv) -> Self {
+        let path = env.base_data_dir.join("attachments.json");
+
+        // Makes no sense to construct this if pageservers aren't going to use it: assume
+        // pageservers have control plane API set
+        let listen_url = env.pageserver.control_plane_api.clone().unwrap();
+
+        let listen = format!(
+            "{}:{}",
+            listen_url.host_str().unwrap(),
+            listen_url.port().unwrap()
+        );
+
+        Self {
+            env: env.clone(),
+            path,
+            listen,
+        }
+    }
+
+    fn pid_file(&self) -> PathBuf {
+        self.env.base_data_dir.join("attachment_service.pid")
+    }
+
+    pub fn start(&self) -> anyhow::Result<Child> {
+        let path_str = self.path.to_string_lossy();
+
+        background_process::start_process(
+            COMMAND,
+            &self.env.base_data_dir,
+            &self.env.attachment_service_bin(),
+            ["-l", &self.listen, "-p", &path_str],
+            [],
+            background_process::InitialPidFile::Create(&self.pid_file()),
+            // TODO: a real status check
+            || Ok(true),
+        )
+    }
+
+    pub fn stop(&self, immediate: bool) -> anyhow::Result<()> {
+        background_process::stop_process(immediate, COMMAND, &self.pid_file())
+    }
+
+    /// Call into the attach_hook API, for use before handing out attachments to pageservers
+    pub fn attach_hook(
+        &self,
+        tenant_id: TenantId,
+        pageserver_id: NodeId,
+    ) -> anyhow::Result<Option<u32>> {
+        use hyper::StatusCode;
+
+        let url = self
+            .env
+            .pageserver
+            .control_plane_api
+            .clone()
+            .unwrap()
+            .join("attach_hook")
+            .unwrap();
+        let client = reqwest::blocking::ClientBuilder::new()
+            .build()
+            .expect("Failed to construct http client");
+
+        let request = AttachHookRequest {
+            tenant_id,
+            pageserver_id: Some(pageserver_id),
+        };
+
+        let response = client.post(url).json(&request).send()?;
+        if response.status() != StatusCode::OK {
+            return Err(anyhow!("Unexpected status {}", response.status()));
+        }
+
+        let response = response.json::<AttachHookResponse>()?;
+        Ok(response.gen)
+    }
+}
diff --git a/control_plane/src/bin/attachment_service.rs b/control_plane/src/bin/attachment_service.rs
new file mode 100644
index 0000000000..e879646b63
--- /dev/null
+++ b/control_plane/src/bin/attachment_service.rs
@@ -0,0 +1,273 @@
+/// The attachment service mimics the aspects of the control plane API
+/// that are required for a pageserver to operate.
+///
+/// This enables running & testing pageservers without a full-blown
+/// deployment of the Neon cloud platform.
+///
+use anyhow::anyhow;
+use clap::Parser;
+use hex::FromHex;
+use hyper::StatusCode;
+use hyper::{Body, Request, Response};
+use serde::{Deserialize, Serialize};
+use std::path::{Path, PathBuf};
+use std::{collections::HashMap, sync::Arc};
+use utils::logging::{self, LogFormat};
+
+use utils::{
+    http::{
+        endpoint::{self},
+        error::ApiError,
+        json::{json_request, json_response},
+        RequestExt, RouterBuilder,
+    },
+    id::{NodeId, TenantId},
+    tcp_listener,
+};
+
+use pageserver_api::control_api::{
+    ReAttachRequest, ReAttachResponse, ReAttachResponseTenant, ValidateRequest, ValidateResponse,
+    ValidateResponseTenant,
+};
+
+use control_plane::attachment_service::{AttachHookRequest, AttachHookResponse};
+
+#[derive(Parser)]
+#[command(author, version, about, long_about = None)]
+#[command(arg_required_else_help(true))]
+struct Cli {
+    /// Host and port to listen on, like `127.0.0.1:1234`
+    #[arg(short, long)]
+    listen: std::net::SocketAddr,
+
+    /// Path to the .json file to store state (will be created if it doesn't exist)
+    #[arg(short, long)]
+    path: PathBuf,
+}
+
+// The persistent state of each Tenant
+#[derive(Serialize, Deserialize, Clone)]
+struct TenantState {
+    // Currently attached pageserver
+    pageserver: Option<NodeId>,
+
+    // Latest generation number: next time we attach, increment this
+    // and use the incremented number when attaching
+    generation: u32,
+}
+
+fn to_hex_map<S, V>(input: &HashMap<TenantId, V>, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: serde::Serializer,
+    V: Clone + Serialize,
+{
+    let transformed = input.iter().map(|(k, v)| (hex::encode(k), v.clone()));
+
+    transformed
+        .collect::<HashMap<String, V>>()
+        .serialize(serializer)
+}
+
+fn from_hex_map<'de, D, V>(deserializer: D) -> Result<HashMap<TenantId, V>, D::Error>
+where
+    D: serde::de::Deserializer<'de>,
+    V: Deserialize<'de>,
+{
+    let hex_map = HashMap::<String, V>::deserialize(deserializer)?;
+    hex_map
+        .into_iter()
+        .map(|(k, v)| {
+            TenantId::from_hex(k)
+                .map(|k| (k, v))
+                .map_err(serde::de::Error::custom)
+        })
+        .collect()
+}
+
+// Top level state available to all HTTP handlers
+#[derive(Serialize, Deserialize)]
+struct PersistentState {
+    #[serde(serialize_with = "to_hex_map", deserialize_with = "from_hex_map")]
+    tenants: HashMap<TenantId, TenantState>,
+
+    #[serde(skip)]
+    path: PathBuf,
+}
+
+impl PersistentState {
+    async fn save(&self) -> anyhow::Result<()> {
+        let bytes = serde_json::to_vec(self)?;
+        tokio::fs::write(&self.path, &bytes).await?;
+
+        Ok(())
+    }
+
+    async fn load(path: &Path) -> anyhow::Result<Self> {
+        let bytes = tokio::fs::read(path).await?;
+        let mut decoded = serde_json::from_slice::<Self>(&bytes)?;
+        decoded.path = path.to_owned();
+        Ok(decoded)
+    }
+
+    async fn load_or_new(path: &Path) -> Self {
+        match Self::load(path).await {
+            Ok(s) => {
+                tracing::info!("Loaded state file at {}", path.display());
+                s
+            }
+            Err(e)
+                if e.downcast_ref::<std::io::Error>()
+                    .map(|e| e.kind() == std::io::ErrorKind::NotFound)
+                    .unwrap_or(false) =>
+            {
+                tracing::info!("Will create state file at {}", path.display());
+                Self {
+                    tenants: HashMap::new(),
+                    path: path.to_owned(),
+                }
+            }
+            Err(e) => {
+                panic!("Failed to load state from '{}': {e:#} (maybe your .neon/ dir was written by an older version?)", path.display())
+            }
+        }
+    }
+}
+
+/// State available to HTTP request handlers
+#[derive(Clone)]
+struct State {
+    inner: Arc<tokio::sync::RwLock<PersistentState>>,
+}
+
+impl State {
+    fn new(persistent_state: PersistentState) -> State {
+        Self {
+            inner: Arc::new(tokio::sync::RwLock::new(persistent_state)),
+        }
+    }
+}
+
+#[inline(always)]
+fn get_state(request: &Request<Body>) -> &State {
+    request
+        .data::<Arc<State>>()
+        .expect("unknown state type")
+        .as_ref()
+}
+
+/// Pageserver calls into this on startup, to learn which tenants it should attach
+async fn handle_re_attach(mut req: Request<Body>) -> Result<Response<Body>, ApiError> {
+    let reattach_req = json_request::<ReAttachRequest>(&mut req).await?;
+
+    let state = get_state(&req).inner.clone();
+    let mut locked = state.write().await;
+
+    let mut response = ReAttachResponse {
+        tenants: Vec::new(),
+    };
+    for (t, state) in &mut locked.tenants {
+        if state.pageserver == Some(reattach_req.node_id) {
+            state.generation += 1;
+            response.tenants.push(ReAttachResponseTenant {
+                id: *t,
+                generation: state.generation,
+            });
+        }
+    }
+
+    locked.save().await.map_err(ApiError::InternalServerError)?;
+
+    json_response(StatusCode::OK, response)
+}
+
+/// Pageserver calls into this before doing deletions, to confirm that it still
+/// holds the latest generation for the tenants with deletions enqueued
+async fn handle_validate(mut req: Request<Body>) -> Result<Response<Body>, ApiError> {
+    let validate_req = json_request::<ValidateRequest>(&mut req).await?;
+
+    let locked = get_state(&req).inner.read().await;
+
+    let mut response = ValidateResponse {
+        tenants: Vec::new(),
+    };
+
+    for req_tenant in validate_req.tenants {
+        if let Some(tenant_state) = locked.tenants.get(&req_tenant.id) {
+            let valid = tenant_state.generation == req_tenant.gen;
+            response.tenants.push(ValidateResponseTenant {
+                id: req_tenant.id,
+                valid,
+            });
+        }
+    }
+
+    json_response(StatusCode::OK, response)
+}
+/// Call into this before attaching a tenant to a pageserver, to acquire a generation number
+/// (in the real control plane this is unnecessary, because the same program is managing
+///  generation numbers and doing attachments).
+async fn handle_attach_hook(mut req: Request<Body>) -> Result<Response<Body>, ApiError> {
+    let attach_req = json_request::<AttachHookRequest>(&mut req).await?;
+
+    let state = get_state(&req).inner.clone();
+    let mut locked = state.write().await;
+
+    let tenant_state = locked
+        .tenants
+        .entry(attach_req.tenant_id)
+        .or_insert_with(|| TenantState {
+            pageserver: attach_req.pageserver_id,
+            generation: 0,
+        });
+
+    if attach_req.pageserver_id.is_some() {
+        tenant_state.generation += 1;
+    }
+    let generation = tenant_state.generation;
+
+    locked.save().await.map_err(ApiError::InternalServerError)?;
+
+    json_response(
+        StatusCode::OK,
+        AttachHookResponse {
+            gen: attach_req.pageserver_id.map(|_| generation),
+        },
+    )
+}
+
+fn make_router(persistent_state: PersistentState) -> RouterBuilder<hyper::Body, ApiError> {
+    endpoint::make_router()
+        .data(Arc::new(State::new(persistent_state)))
+        .post("/re-attach", handle_re_attach)
+        .post("/validate", handle_validate)
+        .post("/attach_hook", handle_attach_hook)
+}
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    logging::init(
+        LogFormat::Plain,
+        logging::TracingErrorLayerEnablement::Disabled,
+    )?;
+
+    let args = Cli::parse();
+    tracing::info!(
+        "Starting, state at {}, listening on {}",
+        args.path.to_string_lossy(),
+        args.listen
+    );
+
+    let persistent_state = PersistentState::load_or_new(&args.path).await;
+
+    let http_listener = tcp_listener::bind(args.listen)?;
+    let router = make_router(persistent_state)
+        .build()
+        .map_err(|err| anyhow!(err))?;
+    let service = utils::http::RouterService::new(router).unwrap();
+    let server = hyper::Server::from_tcp(http_listener)?.serve(service);
+
+    tracing::info!("Serving on {0}", args.listen);
+    server.await?;
+
+    Ok(())
+}
diff --git a/control_plane/src/bin/neon_local.rs b/control_plane/src/bin/neon_local.rs
index ef308cb2d2..6b49b92cfa 100644
--- a/control_plane/src/bin/neon_local.rs
+++ b/control_plane/src/bin/neon_local.rs
@@ -8,6 +8,7 @@
 use anyhow::{anyhow, bail, Context, Result};
 use clap::{value_parser, Arg, ArgAction, ArgMatches, Command};
 use compute_api::spec::ComputeMode;
+use control_plane::attachment_service::AttachmentService;
 use control_plane::endpoint::ComputeControlPlane;
 use control_plane::local_env::LocalEnv;
 use control_plane::pageserver::PageServerNode;
@@ -43,6 +44,8 @@ project_git_version!(GIT_VERSION);
 
 const DEFAULT_PG_VERSION: &str = "15";
 
+const DEFAULT_PAGESERVER_CONTROL_PLANE_API: &str = "http://127.0.0.1:1234/";
+
 fn default_conf() -> String {
     format!(
         r#"
@@ -56,11 +59,13 @@ listen_pg_addr = '{DEFAULT_PAGESERVER_PG_ADDR}'
 listen_http_addr = '{DEFAULT_PAGESERVER_HTTP_ADDR}'
 pg_auth_type = '{trust_auth}'
 http_auth_type = '{trust_auth}'
+control_plane_api = '{DEFAULT_PAGESERVER_CONTROL_PLANE_API}'
 
 [[safekeepers]]
 id = {DEFAULT_SAFEKEEPER_ID}
 pg_port = {DEFAULT_SAFEKEEPER_PG_PORT}
 http_port = {DEFAULT_SAFEKEEPER_HTTP_PORT}
+
 "#,
         trust_auth = AuthType::Trust,
     )
@@ -107,6 +112,7 @@ fn main() -> Result<()> {
             "start" => handle_start_all(sub_args, &env),
             "stop" => handle_stop_all(sub_args, &env),
             "pageserver" => handle_pageserver(sub_args, &env),
+            "attachment_service" => handle_attachment_service(sub_args, &env),
             "safekeeper" => handle_safekeeper(sub_args, &env),
             "endpoint" => handle_endpoint(sub_args, &env),
             "pg" => bail!("'pg' subcommand has been renamed to 'endpoint'"),
@@ -342,13 +348,25 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> an
             }
         }
         Some(("create", create_match)) => {
-            let initial_tenant_id = parse_tenant_id(create_match)?;
             let tenant_conf: HashMap<_, _> = create_match
                 .get_many::<String>("config")
                 .map(|vals| vals.flat_map(|c| c.split_once(':')).collect())
                 .unwrap_or_default();
-            let new_tenant_id = pageserver.tenant_create(initial_tenant_id, tenant_conf)?;
-            println!("tenant {new_tenant_id} successfully created on the pageserver");
+
+            // If tenant ID was not specified, generate one
+            let tenant_id = parse_tenant_id(create_match)?.unwrap_or_else(TenantId::generate);
+
+            let generation = if env.pageserver.control_plane_api.is_some() {
+                // We must register the tenant with the attachment service, so
+                // that when the pageserver restarts, it will be re-attached.
+                let attachment_service = AttachmentService::from_env(env);
+                attachment_service.attach_hook(tenant_id, env.pageserver.id)?
+            } else {
+                None
+            };
+
+            pageserver.tenant_create(tenant_id, generation, tenant_conf)?;
+            println!("tenant {tenant_id} successfully created on the pageserver");
 
             // Create an initial timeline for the new tenant
             let new_timeline_id = parse_timeline_id(create_match)?;
@@ -358,7 +376,7 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> an
                 .context("Failed to parse postgres version from the argument string")?;
 
             let timeline_info = pageserver.timeline_create(
-                new_tenant_id,
+                tenant_id,
                 new_timeline_id,
                 None,
                 None,
@@ -369,17 +387,17 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> an
 
             env.register_branch_mapping(
                 DEFAULT_BRANCH_NAME.to_string(),
-                new_tenant_id,
+                tenant_id,
                 new_timeline_id,
             )?;
 
             println!(
-                "Created an initial timeline '{new_timeline_id}' at Lsn {last_record_lsn} for tenant: {new_tenant_id}",
+                "Created an initial timeline '{new_timeline_id}' at Lsn {last_record_lsn} for tenant: {tenant_id}",
             );
 
             if create_match.get_flag("set-default") {
-                println!("Setting tenant {new_tenant_id} as a default one");
-                env.default_tenant_id = Some(new_tenant_id);
+                println!("Setting tenant {tenant_id} as a default one");
+                env.default_tenant_id = Some(tenant_id);
             }
         }
         Some(("set-default", set_default_match)) => {
@@ -817,6 +835,33 @@ fn handle_pageserver(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul
     Ok(())
 }
 
+fn handle_attachment_service(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> {
+    let svc = AttachmentService::from_env(env);
+    match sub_match.subcommand() {
+        Some(("start", _start_match)) => {
+            if let Err(e) = svc.start() {
+                eprintln!("start failed: {e}");
+                exit(1);
+            }
+        }
+
+        Some(("stop", stop_match)) => {
+            let immediate = stop_match
+                .get_one::<String>("stop-mode")
+                .map(|s| s.as_str())
+                == Some("immediate");
+
+            if let Err(e) = svc.stop(immediate) {
+                eprintln!("stop failed: {}", e);
+                exit(1);
+            }
+        }
+        Some((sub_name, _)) => bail!("Unexpected attachment_service subcommand '{}'", sub_name),
+        None => bail!("no attachment_service subcommand provided"),
+    }
+    Ok(())
+}
+
 fn get_safekeeper(env: &local_env::LocalEnv, id: NodeId) -> Result<SafekeeperNode> {
     if let Some(node) = env.safekeepers.iter().find(|node| node.id == id) {
         Ok(SafekeeperNode::from_env(env, node))
@@ -897,6 +942,16 @@ fn handle_start_all(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> anyhow
 
     broker::start_broker_process(env)?;
 
+    // Only start the attachment service if the pageserver is configured to need it
+    if env.pageserver.control_plane_api.is_some() {
+        let attachment_service = AttachmentService::from_env(env);
+        if let Err(e) = attachment_service.start() {
+            eprintln!("attachment_service start failed: {:#}", e);
+            try_stop_all(env, true);
+            exit(1);
+        }
+    }
+
     let pageserver = PageServerNode::from_env(env);
     if let Err(e) = pageserver.start(&pageserver_config_overrides(sub_match)) {
         eprintln!("pageserver {} start failed: {:#}", env.pageserver.id, e);
@@ -955,6 +1010,13 @@ fn try_stop_all(env: &local_env::LocalEnv, immediate: bool) {
     if let Err(e) = broker::stop_broker_process(env) {
         eprintln!("neon broker stop failed: {e:#}");
     }
+
+    if env.pageserver.control_plane_api.is_some() {
+        let attachment_service = AttachmentService::from_env(env);
+        if let Err(e) = attachment_service.stop(immediate) {
+            eprintln!("attachment service stop failed: {e:#}");
+        }
+    }
 }
 
 fn cli() -> Command {
@@ -1138,6 +1200,14 @@ fn cli() -> Command {
                             .arg(stop_mode_arg.clone()))
                 .subcommand(Command::new("restart").about("Restart local pageserver").arg(pageserver_config_args.clone()))
         )
+        .subcommand(
+            Command::new("attachment_service")
+                .arg_required_else_help(true)
+                .about("Manage attachment_service")
+                .subcommand(Command::new("start").about("Start local pageserver").arg(pageserver_config_args.clone()))
+                .subcommand(Command::new("stop").about("Stop local pageserver")
+                            .arg(stop_mode_arg.clone()))
+        )
         .subcommand(
             Command::new("safekeeper")
                 .arg_required_else_help(true)
diff --git a/control_plane/src/lib.rs b/control_plane/src/lib.rs
index a773b8dcc3..7592880402 100644
--- a/control_plane/src/lib.rs
+++ b/control_plane/src/lib.rs
@@ -7,6 +7,7 @@
 // local installations.
 //
 
+pub mod attachment_service;
 mod background_process;
 pub mod broker;
 pub mod endpoint;
diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs
index 9e42c2e333..0215ab1bb5 100644
--- a/control_plane/src/local_env.rs
+++ b/control_plane/src/local_env.rs
@@ -118,6 +118,9 @@ pub struct PageServerConf {
     // auth type used for the PG and HTTP ports
     pub pg_auth_type: AuthType,
     pub http_auth_type: AuthType,
+
+    // Control plane location
+    pub control_plane_api: Option<Url>,
 }
 
 impl Default for PageServerConf {
@@ -128,6 +131,7 @@ impl Default for PageServerConf {
             listen_http_addr: String::new(),
             pg_auth_type: AuthType::Trust,
             http_auth_type: AuthType::Trust,
+            control_plane_api: None,
         }
     }
 }
@@ -202,6 +206,10 @@ impl LocalEnv {
         self.neon_distrib_dir.join("pageserver")
     }
 
+    pub fn attachment_service_bin(&self) -> PathBuf {
+        self.neon_distrib_dir.join("attachment_service")
+    }
+
     pub fn safekeeper_bin(&self) -> PathBuf {
         self.neon_distrib_dir.join("safekeeper")
     }
diff --git a/control_plane/src/pageserver.rs b/control_plane/src/pageserver.rs
index 2ff09021e5..eecc2479ff 100644
--- a/control_plane/src/pageserver.rs
+++ b/control_plane/src/pageserver.rs
@@ -126,6 +126,13 @@ impl PageServerNode {
             broker_endpoint_param,
         ];
 
+        if let Some(control_plane_api) = &self.env.pageserver.control_plane_api {
+            overrides.push(format!(
+                "control_plane_api='{}'",
+                control_plane_api.as_str()
+            ));
+        }
+
         if self.env.pageserver.http_auth_type != AuthType::Trust
             || self.env.pageserver.pg_auth_type != AuthType::Trust
         {
@@ -316,7 +323,8 @@ impl PageServerNode {
 
     pub fn tenant_create(
         &self,
-        new_tenant_id: Option<TenantId>,
+        new_tenant_id: TenantId,
+        generation: Option<u32>,
         settings: HashMap<&str, &str>,
     ) -> anyhow::Result<TenantId> {
         let mut settings = settings.clone();
@@ -382,11 +390,9 @@ impl PageServerNode {
                 .context("Failed to parse 'gc_feedback' as bool")?,
         };
 
-        // If tenant ID was not specified, generate one
-        let new_tenant_id = new_tenant_id.unwrap_or(TenantId::generate());
-
         let request = models::TenantCreateRequest {
             new_tenant_id,
+            generation,
             config,
         };
         if !settings.is_empty() {
diff --git a/libs/pageserver_api/src/control_api.rs b/libs/pageserver_api/src/control_api.rs
new file mode 100644
index 0000000000..a54fee47a5
--- /dev/null
+++ b/libs/pageserver_api/src/control_api.rs
@@ -0,0 +1,52 @@
+//! Types in this file are for pageserver's upward-facing API calls to the control plane,
+//! required for acquiring and validating tenant generation numbers.
+//!
+//! See docs/rfcs/025-generation-numbers.md
+
+use serde::{Deserialize, Serialize};
+use serde_with::{serde_as, DisplayFromStr};
+use utils::id::{NodeId, TenantId};
+
+#[derive(Serialize, Deserialize)]
+pub struct ReAttachRequest {
+    pub node_id: NodeId,
+}
+
+#[serde_as]
+#[derive(Serialize, Deserialize)]
+pub struct ReAttachResponseTenant {
+    #[serde_as(as = "DisplayFromStr")]
+    pub id: TenantId,
+    pub generation: u32,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct ReAttachResponse {
+    pub tenants: Vec<ReAttachResponseTenant>,
+}
+
+#[serde_as]
+#[derive(Serialize, Deserialize)]
+pub struct ValidateRequestTenant {
+    #[serde_as(as = "DisplayFromStr")]
+    pub id: TenantId,
+    pub gen: u32,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct ValidateRequest {
+    pub tenants: Vec<ValidateRequestTenant>,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct ValidateResponse {
+    pub tenants: Vec<ValidateResponseTenant>,
+}
+
+#[serde_as]
+#[derive(Serialize, Deserialize)]
+pub struct ValidateResponseTenant {
+    #[serde_as(as = "DisplayFromStr")]
+    pub id: TenantId,
+    pub valid: bool,
+}
diff --git a/libs/pageserver_api/src/lib.rs b/libs/pageserver_api/src/lib.rs
index 4890d54f36..d844021785 100644
--- a/libs/pageserver_api/src/lib.rs
+++ b/libs/pageserver_api/src/lib.rs
@@ -1,6 +1,7 @@
 use const_format::formatcp;
 
 /// Public API types
+pub mod control_api;
 pub mod models;
 pub mod reltag;
 
diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs
index 2f4c21326e..340463afa7 100644
--- a/libs/pageserver_api/src/models.rs
+++ b/libs/pageserver_api/src/models.rs
@@ -194,10 +194,22 @@ pub struct TimelineCreateRequest {
 pub struct TenantCreateRequest {
     #[serde_as(as = "DisplayFromStr")]
     pub new_tenant_id: TenantId,
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub generation: Option<u32>,
     #[serde(flatten)]
     pub config: TenantConfig, // as we have a flattened field, we should reject all unknown fields in it
 }
 
+#[serde_as]
+#[derive(Deserialize, Debug)]
+#[serde(deny_unknown_fields)]
+pub struct TenantLoadRequest {
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub generation: Option<u32>,
+}
+
 impl std::ops::Deref for TenantCreateRequest {
     type Target = TenantConfig;
 
@@ -241,15 +253,6 @@ pub struct StatusResponse {
     pub id: NodeId,
 }
 
-impl TenantCreateRequest {
-    pub fn new(new_tenant_id: TenantId) -> TenantCreateRequest {
-        TenantCreateRequest {
-            new_tenant_id,
-            config: TenantConfig::default(),
-        }
-    }
-}
-
 #[serde_as]
 #[derive(Serialize, Deserialize, Debug)]
 #[serde(deny_unknown_fields)]
@@ -293,9 +296,11 @@ impl TenantConfigRequest {
     }
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Deserialize)]
 pub struct TenantAttachRequest {
     pub config: TenantAttachConfig,
+    #[serde(default)]
+    pub generation: Option<u32>,
 }
 
 /// Newtype to enforce deny_unknown_fields on TenantConfig for
diff --git a/libs/remote_storage/src/local_fs.rs b/libs/remote_storage/src/local_fs.rs
index f1095ad8b8..5040183045 100644
--- a/libs/remote_storage/src/local_fs.rs
+++ b/libs/remote_storage/src/local_fs.rs
@@ -148,21 +148,55 @@ impl RemoteStorage for LocalFs {
             Some(folder) => folder.with_base(&self.storage_root),
             None => self.storage_root.clone(),
         };
-        let mut files = vec![];
-        let mut directory_queue = vec![full_path.clone()];
 
+        // If we were given a directory, we may use it as our starting point.
+        // Otherwise, we must go up to the parent directory.  This is because
+        // S3 object list prefixes can be arbitrary strings, but when reading
+        // the local filesystem we need a directory to start calling read_dir on.
+        let mut initial_dir = full_path.clone();
+        match fs::metadata(full_path.clone()).await {
+            Ok(meta) => {
+                if !meta.is_dir() {
+                    // It's not a directory: strip back to the parent
+                    initial_dir.pop();
+                }
+            }
+            Err(e) if e.kind() == ErrorKind::NotFound => {
+                // It's not a file that exists: strip the prefix back to the parent directory
+                initial_dir.pop();
+            }
+            Err(e) => {
+                // Unexpected I/O error
+                anyhow::bail!(e)
+            }
+        }
+
+        // Note that PathBuf starts_with only considers full path segments, but
+        // object prefixes are arbitrary strings, so we need the strings for doing
+        // starts_with later.
+        let prefix = full_path.to_string_lossy();
+
+        let mut files = vec![];
+        let mut directory_queue = vec![initial_dir.clone()];
         while let Some(cur_folder) = directory_queue.pop() {
             let mut entries = fs::read_dir(cur_folder.clone()).await?;
             while let Some(entry) = entries.next_entry().await? {
                 let file_name: PathBuf = entry.file_name().into();
                 let full_file_name = cur_folder.clone().join(&file_name);
-                let file_remote_path = self.local_file_to_relative_path(full_file_name.clone());
-                files.push(file_remote_path.clone());
-                if full_file_name.is_dir() {
-                    directory_queue.push(full_file_name);
+                if full_file_name
+                    .to_str()
+                    .map(|s| s.starts_with(prefix.as_ref()))
+                    .unwrap_or(false)
+                {
+                    let file_remote_path = self.local_file_to_relative_path(full_file_name.clone());
+                    files.push(file_remote_path.clone());
+                    if full_file_name.is_dir() {
+                        directory_queue.push(full_file_name);
+                    }
                 }
             }
         }
+
         Ok(files)
     }
 
diff --git a/libs/utils/src/generation.rs b/libs/utils/src/generation.rs
index 87c6361255..163c8c0467 100644
--- a/libs/utils/src/generation.rs
+++ b/libs/utils/src/generation.rs
@@ -53,6 +53,7 @@ impl Generation {
         matches!(self, Self::None)
     }
 
+    #[track_caller]
     pub fn get_suffix(&self) -> String {
         match self {
             Self::Valid(v) => {
@@ -64,6 +65,30 @@ impl Generation {
             }
         }
     }
+
+    /// `suffix` is the part after "-" in a key
+    ///
+    /// Returns None if parsing was unsuccessful
+    pub fn parse_suffix(suffix: &str) -> Option<Generation> {
+        u32::from_str_radix(suffix, 16).map(Generation::new).ok()
+    }
+
+    #[track_caller]
+    pub fn previous(&self) -> Generation {
+        match self {
+            Self::Valid(n) => {
+                if *n == 0 {
+                    // Since a tenant may be upgraded from a pre-generations state, interpret the "previous" generation
+                    // to 0 as being "no generation".
+                    Self::None
+                } else {
+                    Self::Valid(n - 1)
+                }
+            }
+            Self::None => Self::None,
+            Self::Broken => panic!("Attempted to use a broken generation"),
+        }
+    }
 }
 
 impl Serialize for Generation {
diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs
index 71e3a0ff3f..bf0ad760cb 100644
--- a/pageserver/src/bin/pageserver.rs
+++ b/pageserver/src/bin/pageserver.rs
@@ -388,6 +388,7 @@ fn start_pageserver(
             remote_storage: remote_storage.clone(),
         },
         order,
+        shutdown_pageserver.clone(),
     ))?;
 
     BACKGROUND_RUNTIME.spawn({
diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs
index ab485c969d..624503a3a5 100644
--- a/pageserver/src/config.rs
+++ b/pageserver/src/config.rs
@@ -205,6 +205,8 @@ pub struct PageServerConf {
     /// has it's initial logical size calculated. Not running background tasks for some seconds is
     /// not terrible.
     pub background_task_maximum_delay: Duration,
+
+    pub control_plane_api: Option<Url>,
 }
 
 /// We do not want to store this in a PageServerConf because the latter may be logged
@@ -279,6 +281,8 @@ struct PageServerConfigBuilder {
     ondemand_download_behavior_treat_error_as_warn: BuilderValue<bool>,
 
     background_task_maximum_delay: BuilderValue<Duration>,
+
+    control_plane_api: BuilderValue<Option<Url>>,
 }
 
 impl Default for PageServerConfigBuilder {
@@ -341,6 +345,8 @@ impl Default for PageServerConfigBuilder {
                 DEFAULT_BACKGROUND_TASK_MAXIMUM_DELAY,
             )
             .unwrap()),
+
+            control_plane_api: Set(None),
         }
     }
 }
@@ -469,6 +475,10 @@ impl PageServerConfigBuilder {
         self.background_task_maximum_delay = BuilderValue::Set(delay);
     }
 
+    pub fn control_plane_api(&mut self, api: Url) {
+        self.control_plane_api = BuilderValue::Set(Some(api))
+    }
+
     pub fn build(self) -> anyhow::Result<PageServerConf> {
         let concurrent_tenant_size_logical_size_queries = self
             .concurrent_tenant_size_logical_size_queries
@@ -554,6 +564,9 @@ impl PageServerConfigBuilder {
             background_task_maximum_delay: self
                 .background_task_maximum_delay
                 .ok_or(anyhow!("missing background_task_maximum_delay"))?,
+            control_plane_api: self
+                .control_plane_api
+                .ok_or(anyhow!("missing control_plane_api"))?,
         })
     }
 }
@@ -742,6 +755,7 @@ impl PageServerConf {
                 },
                 "ondemand_download_behavior_treat_error_as_warn" => builder.ondemand_download_behavior_treat_error_as_warn(parse_toml_bool(key, item)?),
                 "background_task_maximum_delay" => builder.background_task_maximum_delay(parse_toml_duration(key, item)?),
+                "control_plane_api" => builder.control_plane_api(parse_toml_string(key, item)?.parse().context("failed to parse control plane URL")?),
                 _ => bail!("unrecognized pageserver option '{key}'"),
             }
         }
@@ -910,6 +924,7 @@ impl PageServerConf {
             test_remote_failures: 0,
             ondemand_download_behavior_treat_error_as_warn: false,
             background_task_maximum_delay: Duration::ZERO,
+            control_plane_api: None,
         }
     }
 }
@@ -1133,6 +1148,7 @@ background_task_maximum_delay = '334 s'
                 background_task_maximum_delay: humantime::parse_duration(
                     defaults::DEFAULT_BACKGROUND_TASK_MAXIMUM_DELAY
                 )?,
+                control_plane_api: None
             },
             "Correct defaults should be used when no config values are provided"
         );
@@ -1188,6 +1204,7 @@ background_task_maximum_delay = '334 s'
                 test_remote_failures: 0,
                 ondemand_download_behavior_treat_error_as_warn: false,
                 background_task_maximum_delay: Duration::from_secs(334),
+                control_plane_api: None
             },
             "Should be able to parse all basic config values correctly"
         );
diff --git a/pageserver/src/control_plane_client.rs b/pageserver/src/control_plane_client.rs
new file mode 100644
index 0000000000..192eb16789
--- /dev/null
+++ b/pageserver/src/control_plane_client.rs
@@ -0,0 +1,119 @@
+use std::collections::HashMap;
+
+use hyper::StatusCode;
+use pageserver_api::control_api::{ReAttachRequest, ReAttachResponse};
+use tokio_util::sync::CancellationToken;
+use url::Url;
+use utils::{
+    backoff,
+    generation::Generation,
+    id::{NodeId, TenantId},
+};
+
+use crate::config::PageServerConf;
+
+// Backoffs when control plane requests do not succeed: compromise between reducing load
+// on control plane, and retrying frequently when we are blocked on a control plane
+// response to make progress.
+const BACKOFF_INCREMENT: f64 = 0.1;
+const BACKOFF_MAX: f64 = 10.0;
+
+/// The Pageserver's client for using the control plane API: this is a small subset
+/// of the overall control plane API, for dealing with generations (see docs/rfcs/025-generation-numbers.md)
+pub(crate) struct ControlPlaneClient {
+    http_client: reqwest::Client,
+    base_url: Url,
+    node_id: NodeId,
+    cancel: CancellationToken,
+}
+
+impl ControlPlaneClient {
+    /// A None return value indicates that the input `conf` object does not have control
+    /// plane API enabled.
+    pub(crate) fn new(conf: &'static PageServerConf, cancel: &CancellationToken) -> Option<Self> {
+        let mut url = match conf.control_plane_api.as_ref() {
+            Some(u) => u.clone(),
+            None => return None,
+        };
+
+        if let Ok(mut segs) = url.path_segments_mut() {
+            // This ensures that `url` ends with a slash if it doesn't already.
+            // That way, we can subsequently use join() to safely attach extra path elements.
+            segs.pop_if_empty().push("");
+        }
+
+        let client = reqwest::ClientBuilder::new()
+            .build()
+            .expect("Failed to construct http client");
+
+        Some(Self {
+            http_client: client,
+            base_url: url,
+            node_id: conf.id,
+            cancel: cancel.clone(),
+        })
+    }
+
+    async fn try_re_attach(
+        &self,
+        url: Url,
+        request: &ReAttachRequest,
+    ) -> anyhow::Result<ReAttachResponse> {
+        match self.http_client.post(url).json(request).send().await {
+            Err(e) => Err(anyhow::Error::from(e)),
+            Ok(r) => {
+                if r.status() == StatusCode::OK {
+                    r.json::<ReAttachResponse>()
+                        .await
+                        .map_err(anyhow::Error::from)
+                } else {
+                    Err(anyhow::anyhow!("Unexpected status {}", r.status()))
+                }
+            }
+        }
+    }
+
+    /// Block until we get a successful response
+    pub(crate) async fn re_attach(&self) -> anyhow::Result<HashMap<TenantId, Generation>> {
+        let re_attach_path = self
+            .base_url
+            .join("re-attach")
+            .expect("Failed to build re-attach path");
+        let request = ReAttachRequest {
+            node_id: self.node_id,
+        };
+
+        let mut attempt = 0;
+        loop {
+            let result = self.try_re_attach(re_attach_path.clone(), &request).await;
+            match result {
+                Ok(res) => {
+                    tracing::info!(
+                        "Received re-attach response with {} tenants",
+                        res.tenants.len()
+                    );
+
+                    return Ok(res
+                        .tenants
+                        .into_iter()
+                        .map(|t| (t.id, Generation::new(t.generation)))
+                        .collect::<HashMap<_, _>>());
+                }
+                Err(e) => {
+                    tracing::error!("Error re-attaching tenants, retrying: {e:#}");
+                    backoff::exponential_backoff(
+                        attempt,
+                        BACKOFF_INCREMENT,
+                        BACKOFF_MAX,
+                        &self.cancel,
+                    )
+                    .await;
+                    if self.cancel.is_cancelled() {
+                        return Err(anyhow::anyhow!("Shutting down"));
+                    }
+                    attempt += 1;
+                }
+            }
+        }
+    }
+}
diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml
index 38e07f172d..4988641d6a 100644
--- a/pageserver/src/http/openapi_spec.yml
+++ b/pageserver/src/http/openapi_spec.yml
@@ -383,7 +383,6 @@ paths:
         schema:
           type: string
           format: hex
-
     post:
       description: |
         Schedules attach operation to happen in the background for the given tenant.
@@ -1020,6 +1019,9 @@ components:
       properties:
         config:
           $ref: '#/components/schemas/TenantConfig'
+        generation:
+          type: integer
+          description: Attachment generation number.
     TenantConfigRequest:
       allOf:
         - $ref: '#/components/schemas/TenantConfig'
diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs
index f86657fa77..286aa3dfb0 100644
--- a/pageserver/src/http/routes.rs
+++ b/pageserver/src/http/routes.rs
@@ -8,7 +8,9 @@ use anyhow::{anyhow, Context, Result};
 use hyper::StatusCode;
 use hyper::{Body, Request, Response, Uri};
 use metrics::launch_timestamp::LaunchTimestamp;
-use pageserver_api::models::{DownloadRemoteLayersTaskSpawnRequest, TenantAttachRequest};
+use pageserver_api::models::{
+    DownloadRemoteLayersTaskSpawnRequest, TenantAttachRequest, TenantLoadRequest,
+};
 use remote_storage::GenericRemoteStorage;
 use storage_broker::BrokerClientChannel;
 use tenant_size_model::{SizeResult, StorageModel};
@@ -32,11 +34,13 @@ use crate::tenant::mgr::{
 };
 use crate::tenant::size::ModelInputs;
 use crate::tenant::storage_layer::LayerAccessStatsReset;
-use crate::tenant::{LogicalSizeCalculationCause, PageReconstructError, Timeline};
+use crate::tenant::timeline::Timeline;
+use crate::tenant::{LogicalSizeCalculationCause, PageReconstructError};
 use crate::{config::PageServerConf, tenant::mgr};
 use crate::{disk_usage_eviction_task, tenant};
 use utils::{
     auth::JwtAuth,
+    generation::Generation,
     http::{
         endpoint::{self, attach_openapi_ui, auth_middleware, check_permission_with},
         error::{ApiError, HttpErrorBody},
@@ -472,7 +476,7 @@ async fn tenant_attach_handler(
     check_permission(&request, Some(tenant_id))?;
 
     let maybe_body: Option<TenantAttachRequest> = json_request_or_empty_body(&mut request).await?;
-    let tenant_conf = match maybe_body {
+    let tenant_conf = match &maybe_body {
         Some(request) => TenantConfOpt::try_from(&*request.config).map_err(ApiError::BadRequest)?,
         None => TenantConfOpt::default(),
     };
@@ -483,10 +487,13 @@ async fn tenant_attach_handler(
 
     let state = get_state(&request);
 
+    let generation = get_request_generation(state, maybe_body.as_ref().and_then(|r| r.generation))?;
+
     if let Some(remote_storage) = &state.remote_storage {
         mgr::attach_tenant(
             state.conf,
             tenant_id,
+            generation,
             tenant_conf,
             state.broker_client.clone(),
             remote_storage.clone(),
@@ -538,7 +545,7 @@ async fn tenant_detach_handler(
 }
 
 async fn tenant_load_handler(
-    request: Request<Body>,
+    mut request: Request<Body>,
     _cancel: CancellationToken,
 ) -> Result<Response<Body>, ApiError> {
     let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
@@ -546,10 +553,18 @@ async fn tenant_load_handler(
 
     let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn);
 
+    let maybe_body: Option<TenantLoadRequest> = json_request_or_empty_body(&mut request).await?;
+
     let state = get_state(&request);
+
+    // The /load request is only usable when control_plane_api is not set.  Once it is set, callers
+    // should always use /attach instead.
+    let generation = get_request_generation(state, maybe_body.as_ref().and_then(|r| r.generation))?;
+
     mgr::load_tenant(
         state.conf,
         tenant_id,
+        generation,
         state.broker_client.clone(),
         state.remote_storage.clone(),
         &ctx,
@@ -851,6 +866,21 @@ pub fn html_response(status: StatusCode, data: String) -> Result<Response<Body>,
     Ok(response)
 }
 
+/// Helper for requests that may take a generation, which is mandatory
+/// when control_plane_api is set, but otherwise defaults to Generation::none()
+fn get_request_generation(state: &State, req_gen: Option<u32>) -> Result<Generation, ApiError> {
+    if state.conf.control_plane_api.is_some() {
+        req_gen
+            .map(Generation::new)
+            .ok_or(ApiError::BadRequest(anyhow!(
+                "generation attribute missing"
+            )))
+    } else {
+        // Legacy mode: all tenants operate with no generation
+        Ok(Generation::none())
+    }
+}
+
 async fn tenant_create_handler(
     mut request: Request<Body>,
     _cancel: CancellationToken,
@@ -867,14 +897,17 @@ async fn tenant_create_handler(
     let tenant_conf =
         TenantConfOpt::try_from(&request_data.config).map_err(ApiError::BadRequest)?;
 
-    let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn);
-
     let state = get_state(&request);
 
+    let generation = get_request_generation(state, request_data.generation)?;
+
+    let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn);
+
     let new_tenant = mgr::create_tenant(
         state.conf,
         tenant_conf,
         target_tenant_id,
+        generation,
         state.broker_client.clone(),
         state.remote_storage.clone(),
         &ctx,
diff --git a/pageserver/src/lib.rs b/pageserver/src/lib.rs
index cb20caba1f..3049ad6a4e 100644
--- a/pageserver/src/lib.rs
+++ b/pageserver/src/lib.rs
@@ -3,6 +3,7 @@ pub mod basebackup;
 pub mod config;
 pub mod consumption_metrics;
 pub mod context;
+mod control_plane_client;
 pub mod disk_usage_eviction_task;
 pub mod http;
 pub mod import_datadir;
diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs
index 02df8af51c..f0639844bd 100644
--- a/pageserver/src/tenant.rs
+++ b/pageserver/src/tenant.rs
@@ -3381,7 +3381,7 @@ pub mod harness {
         pub tenant_conf: TenantConf,
         pub tenant_id: TenantId,
         pub generation: Generation,
-        remote_storage: GenericRemoteStorage,
+        pub remote_storage: GenericRemoteStorage,
         pub remote_fs_dir: PathBuf,
     }
 
diff --git a/pageserver/src/tenant/metadata.rs b/pageserver/src/tenant/metadata.rs
index 7b05704e4f..75ffe09696 100644
--- a/pageserver/src/tenant/metadata.rs
+++ b/pageserver/src/tenant/metadata.rs
@@ -230,6 +230,23 @@ impl TimelineMetadata {
     pub fn pg_version(&self) -> u32 {
         self.body.pg_version
     }
+
+    // Checksums make it awkward to build a valid instance by hand.  This helper
+    // provides a TimelineMetadata with a valid checksum in its header.
+    #[cfg(test)]
+    pub fn example() -> Self {
+        let instance = Self::new(
+            "0/16960E8".parse::<Lsn>().unwrap(),
+            None,
+            None,
+            Lsn::from_hex("00000000").unwrap(),
+            Lsn::from_hex("00000000").unwrap(),
+            Lsn::from_hex("00000000").unwrap(),
+            0,
+        );
+        let bytes = instance.to_bytes().unwrap();
+        Self::from_bytes(&bytes).unwrap()
+    }
 }
 
 impl<'de> Deserialize<'de> for TimelineMetadata {
diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs
index 72d150e0eb..b92c1f69c3 100644
--- a/pageserver/src/tenant/mgr.rs
+++ b/pageserver/src/tenant/mgr.rs
@@ -11,6 +11,7 @@ use anyhow::Context;
 use once_cell::sync::Lazy;
 use tokio::sync::RwLock;
 use tokio::task::JoinSet;
+use tokio_util::sync::CancellationToken;
 use tracing::*;
 
 use remote_storage::GenericRemoteStorage;
@@ -18,6 +19,7 @@ use utils::crashsafe;
 
 use crate::config::PageServerConf;
 use crate::context::{DownloadBehavior, RequestContext};
+use crate::control_plane_client::ControlPlaneClient;
 use crate::task_mgr::{self, TaskKind};
 use crate::tenant::config::TenantConfOpt;
 use crate::tenant::delete::DeleteTenantFlow;
@@ -94,12 +96,21 @@ pub async fn init_tenant_mgr(
     conf: &'static PageServerConf,
     resources: TenantSharedResources,
     init_order: InitializationOrder,
+    cancel: CancellationToken,
 ) -> anyhow::Result<()> {
     // Scan local filesystem for attached tenants
     let tenants_dir = conf.tenants_path();
 
     let mut tenants = HashMap::new();
 
+    // If we are configured to use the control plane API, then it is the source of truth for what tenants to load.
+    let tenant_generations = if let Some(client) = ControlPlaneClient::new(conf, &cancel) {
+        Some(client.re_attach().await?)
+    } else {
+        info!("Control plane API not configured, tenant generations are disabled");
+        None
+    };
+
     let mut dir_entries = fs::read_dir(&tenants_dir)
         .await
         .with_context(|| format!("Failed to list tenants dir {tenants_dir:?}"))?;
@@ -149,9 +160,53 @@ pub async fn init_tenant_mgr(
                         continue;
                     }
 
+                    let tenant_id = match tenant_dir_path
+                        .file_name()
+                        .and_then(OsStr::to_str)
+                        .unwrap_or_default()
+                        .parse::<TenantId>()
+                    {
+                        Ok(id) => id,
+                        Err(_) => {
+                            warn!(
+                                "Invalid tenant path (garbage in our repo directory?): {}",
+                                tenant_dir_path.display()
+                            );
+                            continue;
+                        }
+                    };
+
+                    let generation = if let Some(generations) = &tenant_generations {
+                        // We have a generation map: treat it as the authority for whether
+                        // this tenant is really attached.
+                        if let Some(gen) = generations.get(&tenant_id) {
+                            *gen
+                        } else {
+                            info!("Detaching tenant {tenant_id}, control plane omitted it in re-attach response");
+                            if let Err(e) = safe_remove_tenant_dir_all(&tenant_dir_path).await {
+                                error!(
+                                    "Failed to remove detached tenant directory '{}': {:?}",
+                                    tenant_dir_path.display(),
+                                    e
+                                );
+                            }
+                            continue;
+                        }
+                    } else {
+                        // Legacy mode: no generation information, any tenant present
+                        // on local disk may activate
+                        info!(
+                            "Starting tenant {} in legacy mode, no generation",
+                            tenant_dir_path.display()
+                        );
+                        Generation::none()
+                    };
+
                     match schedule_local_tenant_processing(
                         conf,
+                        tenant_id,
                         &tenant_dir_path,
+                        generation,
                         resources.clone(),
                         Some(init_order.clone()),
                         &TENANTS,
@@ -185,9 +240,12 @@ pub async fn init_tenant_mgr(
     Ok(())
 }
 
+#[allow(clippy::too_many_arguments)]
 pub(crate) fn schedule_local_tenant_processing(
     conf: &'static PageServerConf,
+    tenant_id: TenantId,
     tenant_path: &Path,
+    generation: Generation,
     resources: TenantSharedResources,
     init_order: Option<InitializationOrder>,
     tenants: &'static tokio::sync::RwLock<TenantsMap>,
@@ -208,15 +266,6 @@ pub(crate) fn schedule_local_tenant_processing(
         "Cannot load tenant from empty directory {tenant_path:?}"
     );
 
-    let tenant_id = tenant_path
-        .file_name()
-        .and_then(OsStr::to_str)
-        .unwrap_or_default()
-        .parse::<TenantId>()
-        .with_context(|| {
-            format!("Could not parse tenant id out of the tenant dir name in path {tenant_path:?}")
-        })?;
-
     let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_id);
     anyhow::ensure!(
         !conf.tenant_ignore_mark_file_path(&tenant_id).exists(),
@@ -229,7 +278,7 @@ pub(crate) fn schedule_local_tenant_processing(
             match Tenant::spawn_attach(
                 conf,
                 tenant_id,
-                Generation::none(),
+                generation,
                 resources.broker_client,
                 tenants,
                 remote_storage,
@@ -253,13 +302,7 @@ pub(crate) fn schedule_local_tenant_processing(
         info!("tenant {tenant_id} is assumed to be loadable, starting load operation");
         // Start loading the tenant into memory. It will initially be in Loading state.
         Tenant::spawn_load(
-            conf,
-            tenant_id,
-            Generation::none(),
-            resources,
-            init_order,
-            tenants,
-            ctx,
+            conf, tenant_id, generation, resources, init_order, tenants, ctx,
         )
     };
     Ok(tenant)
@@ -383,6 +426,7 @@ pub async fn create_tenant(
     conf: &'static PageServerConf,
     tenant_conf: TenantConfOpt,
     tenant_id: TenantId,
+    generation: Generation,
     broker_client: storage_broker::BrokerClientChannel,
     remote_storage: Option<GenericRemoteStorage>,
     ctx: &RequestContext,
@@ -400,7 +444,8 @@ pub async fn create_tenant(
             remote_storage,
         };
         let created_tenant =
-            schedule_local_tenant_processing(conf, &tenant_directory, tenant_resources, None, &TENANTS, ctx)?;
+            schedule_local_tenant_processing(conf, tenant_id, &tenant_directory,
+                generation, tenant_resources, None, &TENANTS, ctx)?;
         // TODO: tenant object & its background loops remain, untracked in tenant map, if we fail here.
         //      See https://github.com/neondatabase/neon/issues/4233
 
@@ -548,6 +593,7 @@ async fn detach_tenant0(
 pub async fn load_tenant(
     conf: &'static PageServerConf,
     tenant_id: TenantId,
+    generation: Generation,
     broker_client: storage_broker::BrokerClientChannel,
     remote_storage: Option<GenericRemoteStorage>,
     ctx: &RequestContext,
@@ -564,7 +610,7 @@ pub async fn load_tenant(
             broker_client,
             remote_storage,
         };
-        let new_tenant = schedule_local_tenant_processing(conf, &tenant_path,  resources, None,  &TENANTS, ctx)
+        let new_tenant = schedule_local_tenant_processing(conf, tenant_id, &tenant_path, generation, resources, None,  &TENANTS, ctx)
             .with_context(|| {
                 format!("Failed to schedule tenant processing in path {tenant_path:?}")
             })?;
@@ -628,6 +674,7 @@ pub async fn list_tenants() -> Result<Vec<(TenantId, TenantState)>, TenantMapLis
 pub async fn attach_tenant(
     conf: &'static PageServerConf,
     tenant_id: TenantId,
+    generation: Generation,
     tenant_conf: TenantConfOpt,
     broker_client: storage_broker::BrokerClientChannel,
     remote_storage: GenericRemoteStorage,
@@ -649,7 +696,7 @@ pub async fn attach_tenant(
             broker_client,
             remote_storage: Some(remote_storage),
         };
-        let attached_tenant = schedule_local_tenant_processing(conf, &tenant_dir, resources, None, &TENANTS, ctx)?;
+        let attached_tenant = schedule_local_tenant_processing(conf, tenant_id, &tenant_dir, generation, resources, None, &TENANTS, ctx)?;
         // TODO: tenant object & its background loops remain, untracked in tenant map, if we fail here.
         //      See https://github.com/neondatabase/neon/issues/4233
 
diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs
index 13f3fac41c..6f42b54ac2 100644
--- a/pageserver/src/tenant/remote_timeline_client.rs
+++ b/pageserver/src/tenant/remote_timeline_client.rs
@@ -1430,6 +1430,30 @@ pub fn remote_index_path(
     .expect("Failed to construct path")
 }
 
+/// Given the key of an index, parse out the generation part of the name
+pub(crate) fn parse_remote_index_path(path: RemotePath) -> Option<Generation> {
+    let file_name = match path.get_path().file_name() {
+        Some(f) => f,
+        None => {
+            // Unexpected: we should be seeing index_part.json paths only
+            tracing::warn!("Malformed index key {}", path);
+            return None;
+        }
+    };
+
+    let file_name_str = match file_name.to_str() {
+        Some(s) => s,
+        None => {
+            tracing::warn!("Malformed index key {:?}", path);
+            return None;
+        }
+    };
+    match file_name_str.split_once('-') {
+        Some((_, gen_suffix)) => Generation::parse_suffix(gen_suffix),
+        None => None,
+    }
+}
+
 /// Files on the remote storage are stored with paths, relative to the workdir.
 /// That path includes in itself both tenant and timeline ids, allowing to have a unique remote storage path.
 ///
@@ -1546,6 +1570,33 @@ mod tests {
                 tenant_ctx: ctx,
             })
         }
+
+        /// Construct a RemoteTimelineClient in an arbitrary generation
+        fn build_client(&self, generation: Generation) -> Arc<RemoteTimelineClient> {
+            Arc::new(RemoteTimelineClient {
+                conf: self.harness.conf,
+                runtime: tokio::runtime::Handle::current(),
+                tenant_id: self.harness.tenant_id,
+                timeline_id: TIMELINE_ID,
+                generation,
+                storage_impl: self.harness.remote_storage.clone(),
+                upload_queue: Mutex::new(UploadQueue::Uninitialized),
+                metrics: Arc::new(RemoteTimelineClientMetrics::new(
+                    &self.harness.tenant_id,
+                    &TIMELINE_ID,
+                )),
+            })
+        }
+
+        /// A tracing::Span that satisfies remote_timeline_client methods that assert tenant_id
+        /// and timeline_id are present.
+        fn span(&self) -> tracing::Span {
+            tracing::info_span!(
+                "test",
+                tenant_id = %self.harness.tenant_id,
+                timeline_id = %TIMELINE_ID
+            )
+        }
     }
 
     // Test scheduling
@@ -1565,12 +1616,16 @@ mod tests {
         // Schedule another deletion. Check that it's launched immediately.
         // Schedule index upload. Check that it's queued
 
+        let test_setup = TestSetup::new("upload_scheduling").await.unwrap();
+        let span = test_setup.span();
+        let _guard = span.enter();
+
         let TestSetup {
             harness,
             tenant: _tenant,
             timeline,
             tenant_ctx: _tenant_ctx,
-        } = TestSetup::new("upload_scheduling").await.unwrap();
+        } = test_setup;
 
         let client = timeline.remote_client.as_ref().unwrap();
 
@@ -1818,4 +1873,106 @@ mod tests {
             };
         assert_eq!(actual_c, expected_c);
     }
+
+    async fn inject_index_part(test_state: &TestSetup, generation: Generation) -> IndexPart {
+        // An empty IndexPart, just sufficient to ensure deserialization will succeed
+        let example_metadata = TimelineMetadata::example();
+        let example_index_part = IndexPart::new(
+            HashMap::new(),
+            example_metadata.disk_consistent_lsn(),
+            example_metadata,
+        );
+
+        let index_part_bytes = serde_json::to_vec(&example_index_part).unwrap();
+
+        let timeline_path = test_state.harness.timeline_path(&TIMELINE_ID);
+        let remote_timeline_dir = test_state.harness.remote_fs_dir.join(
+            timeline_path
+                .strip_prefix(&test_state.harness.conf.workdir)
+                .unwrap(),
+        );
+
+        std::fs::create_dir_all(remote_timeline_dir).expect("creating test dir should work");
+
+        let index_path = test_state.harness.remote_fs_dir.join(
+            remote_index_path(&test_state.harness.tenant_id, &TIMELINE_ID, generation).get_path(),
+        );
+        eprintln!("Writing {}", index_path.display());
+        std::fs::write(&index_path, index_part_bytes).unwrap();
+        example_index_part
+    }
+
+    /// Assert that when a RemoteTimelineclient in generation `get_generation` fetches its
+    /// index, the IndexPart returned is equal to `expected`
+    async fn assert_got_index_part(
+        test_state: &TestSetup,
+        get_generation: Generation,
+        expected: &IndexPart,
+    ) {
+        let client = test_state.build_client(get_generation);
+
+        let download_r = client
+            .download_index_file()
+            .await
+            .expect("download should always succeed");
+        assert!(matches!(download_r, MaybeDeletedIndexPart::IndexPart(_)));
+        match download_r {
+            MaybeDeletedIndexPart::IndexPart(index_part) => {
+                assert_eq!(&index_part, expected);
+            }
+            MaybeDeletedIndexPart::Deleted(_index_part) => panic!("Test doesn't set deleted_at"),
+        }
+    }
+
+    #[tokio::test]
+    async fn index_part_download_simple() -> anyhow::Result<()> {
+        let test_state = TestSetup::new("index_part_download_simple").await.unwrap();
+        let span = test_state.span();
+        let _guard = span.enter();
+
+        // Simple case: we are in generation N, load the index from generation N - 1
+        let generation_n = 5;
+        let injected = inject_index_part(&test_state, Generation::new(generation_n - 1)).await;
+
+        assert_got_index_part(&test_state, Generation::new(generation_n), &injected).await;
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn index_part_download_ordering() -> anyhow::Result<()> {
+        let test_state = TestSetup::new("index_part_download_ordering")
+            .await
+            .unwrap();
+
+        let span = test_state.span();
+        let _guard = span.enter();
+
+        // A generation-less IndexPart exists in the bucket, we should find it
+        let generation_n = 5;
+        let injected_none = inject_index_part(&test_state, Generation::none()).await;
+        assert_got_index_part(&test_state, Generation::new(generation_n), &injected_none).await;
+
+        // If a more recent-than-none generation exists, we should prefer to load that
+        let injected_1 = inject_index_part(&test_state, Generation::new(1)).await;
+        assert_got_index_part(&test_state, Generation::new(generation_n), &injected_1).await;
+
+        // If a more-recent-than-me generation exists, we should ignore it.
+        let _injected_10 = inject_index_part(&test_state, Generation::new(10)).await;
+        assert_got_index_part(&test_state, Generation::new(generation_n), &injected_1).await;
+
+        // If a directly previous generation exists, _and_ an index exists in my own
+        // generation, I should prefer my own generation.
+        let _injected_prev =
+            inject_index_part(&test_state, Generation::new(generation_n - 1)).await;
+        let injected_current = inject_index_part(&test_state, Generation::new(generation_n)).await;
+        assert_got_index_part(
+            &test_state,
+            Generation::new(generation_n),
+            &injected_current,
+        )
+        .await;
+
+        Ok(())
+    }
 }
diff --git a/pageserver/src/tenant/remote_timeline_client/download.rs b/pageserver/src/tenant/remote_timeline_client/download.rs
index dc8d87b9e1..9863215529 100644
--- a/pageserver/src/tenant/remote_timeline_client/download.rs
+++ b/pageserver/src/tenant/remote_timeline_client/download.rs
@@ -24,7 +24,10 @@ use utils::crashsafe::path_with_suffix_extension;
 use utils::id::{TenantId, TimelineId};
 
 use super::index::{IndexPart, LayerFileMetadata};
-use super::{remote_index_path, FAILED_DOWNLOAD_WARN_THRESHOLD, FAILED_REMOTE_OP_RETRIES};
+use super::{
+    parse_remote_index_path, remote_index_path, FAILED_DOWNLOAD_WARN_THRESHOLD,
+    FAILED_REMOTE_OP_RETRIES,
+};
 
 static MAX_DOWNLOAD_DURATION: Duration = Duration::from_secs(120);
 
@@ -219,13 +222,13 @@ pub async fn list_remote_timelines(
     Ok(timeline_ids)
 }
 
-pub(super) async fn download_index_part(
+async fn do_download_index_part(
     storage: &GenericRemoteStorage,
     tenant_id: &TenantId,
     timeline_id: &TimelineId,
-    generation: Generation,
+    index_generation: Generation,
 ) -> Result<IndexPart, DownloadError> {
-    let remote_path = remote_index_path(tenant_id, timeline_id, generation);
+    let remote_path = remote_index_path(tenant_id, timeline_id, index_generation);
 
     let index_part_bytes = download_retry(
         || async {
@@ -252,6 +255,105 @@ pub(super) async fn download_index_part(
     Ok(index_part)
 }
 
+/// index_part.json objects are suffixed with a generation number, so we cannot
+/// directly GET the latest index part without doing some probing.
+///
+/// In this function we probe for the most recent index in a generation <= our current generation.
+/// See "Finding the remote indices for timelines" in docs/rfcs/025-generation-numbers.md
+#[tracing::instrument(skip_all, fields(generation=?my_generation))]
+pub(super) async fn download_index_part(
+    storage: &GenericRemoteStorage,
+    tenant_id: &TenantId,
+    timeline_id: &TimelineId,
+    my_generation: Generation,
+) -> Result<IndexPart, DownloadError> {
+    debug_assert_current_span_has_tenant_and_timeline_id();
+
+    if my_generation.is_none() {
+        // Operating without generations: just fetch the generation-less path
+        return do_download_index_part(storage, tenant_id, timeline_id, my_generation).await;
+    }
+
+    // Stale case: If we were intentionally attached in a stale generation, there may already be a remote
+    // index in our generation.
+    //
+    // This is an optimization to avoid doing the listing for the general case below.
+    let res = do_download_index_part(storage, tenant_id, timeline_id, my_generation).await;
+    match res {
+        Ok(index_part) => {
+            tracing::debug!(
+                "Found index_part from current generation (this is a stale attachment)"
+            );
+            return Ok(index_part);
+        }
+        Err(DownloadError::NotFound) => {}
+        Err(e) => return Err(e),
+    };
+
+    // Typical case: the previous generation of this tenant was running healthily, and had uploaded
+    // and index part.  We may safely start from this index without doing a listing, because:
+    //  - We checked for current generation case above
+    //  - generations > my_generation are to be ignored
+    //  - any other indices that exist would have an older generation than `previous_gen`, and
+    //    we want to find the most recent index from a previous generation.
+    //
+    // This is an optimization to avoid doing the listing for the general case below.
+    let res =
+        do_download_index_part(storage, tenant_id, timeline_id, my_generation.previous()).await;
+    match res {
+        Ok(index_part) => {
+            tracing::debug!("Found index_part from previous generation");
+            return Ok(index_part);
+        }
+        Err(DownloadError::NotFound) => {
+            tracing::debug!(
+                "No index_part found from previous generation, falling back to listing"
+            );
+        }
+        Err(e) => {
+            return Err(e);
+        }
+    }
+
+    // General case/fallback: if there is no index at my_generation or prev_generation, then list all index_part.json
+    // objects, and select the highest one with a generation <= my_generation.
+    let index_prefix = remote_index_path(tenant_id, timeline_id, Generation::none());
+    let indices = backoff::retry(
+        || async { storage.list_files(Some(&index_prefix)).await },
+        |_| false,
+        FAILED_DOWNLOAD_WARN_THRESHOLD,
+        FAILED_REMOTE_OP_RETRIES,
+        "listing index_part files",
+        // TODO: use a cancellation token (https://github.com/neondatabase/neon/issues/5066)
+        backoff::Cancel::new(CancellationToken::new(), || -> anyhow::Error {
+            unreachable!()
+        }),
+    )
+    .await
+    .map_err(DownloadError::Other)?;
+
+    // General case logic for which index to use: the latest index whose generation
+    // is <= our own.  See "Finding the remote indices for timelines" in docs/rfcs/025-generation-numbers.md
+    let max_previous_generation = indices
+        .into_iter()
+        .filter_map(parse_remote_index_path)
+        .filter(|g| g <= &my_generation)
+        .max();
+
+    match max_previous_generation {
+        Some(g) => {
+            tracing::debug!("Found index_part in generation {g:?}");
+            do_download_index_part(storage, tenant_id, timeline_id, g).await
+        }
+        None => {
+            // Migration from legacy pre-generation state: we have a generation but no prior
+            // attached pageservers did.  Try to load from a no-generation path.
+            tracing::info!("No index_part.json* found");
+            do_download_index_part(storage, tenant_id, timeline_id, Generation::none()).await
+        }
+    }
+}
+
 /// Helper function to handle retries for a download operation.
 ///
 /// Remote operations can fail due to rate limits (IAM, S3), spurious network
diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs
index d7ff70a29c..7b6c6dbfad 100644
--- a/pageserver/src/tenant/timeline.rs
+++ b/pageserver/src/tenant/timeline.rs
@@ -1724,11 +1724,18 @@ impl Timeline {
                 for (name, decision) in decided {
                     let decision = match decision {
                         Ok(UseRemote { local, remote }) => {
-                            path.push(name.file_name());
-                            init::cleanup_local_file_for_remote(&path, &local, &remote)?;
-                            path.pop();
-
-                            UseRemote { local, remote }
+                            // Remote is authoritative, but we may still choose to retain
+                            // the local file if the contents appear to match
+                            if local.file_size() == remote.file_size() {
+                                // Use the local file, but take the remote metadata so that we pick up
+                                // the correct generation.
+                                UseLocal(remote)
+                            } else {
+                                path.push(name.file_name());
+                                init::cleanup_local_file_for_remote(&path, &local, &remote)?;
+                                path.pop();
+                                UseRemote { local, remote }
+                            }
                         }
                         Ok(decision) => decision,
                         Err(FutureLayer { local }) => {
diff --git a/pageserver/src/tenant/timeline/init.rs b/pageserver/src/tenant/timeline/init.rs
index 33effb4318..22976a514d 100644
--- a/pageserver/src/tenant/timeline/init.rs
+++ b/pageserver/src/tenant/timeline/init.rs
@@ -147,11 +147,7 @@ pub(super) fn reconcile(
                 Err(FutureLayer { local })
             } else {
                 Ok(match (local, remote) {
-                    (Some(local), Some(remote)) if local != remote => {
-                        assert_eq!(local.generation, remote.generation);
-
-                        UseRemote { local, remote }
-                    }
+                    (Some(local), Some(remote)) if local != remote => UseRemote { local, remote },
                     (Some(x), Some(_)) => UseLocal(x),
                     (None, Some(x)) => Evicted(x),
                     (Some(x), None) => NeedsUpload(x),
diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py
index 4ea1d79705..a8ca3b50a1 100644
--- a/test_runner/fixtures/neon_fixtures.py
+++ b/test_runner/fixtures/neon_fixtures.py
@@ -456,6 +456,7 @@ class NeonEnvBuilder:
         self.preserve_database_files = preserve_database_files
         self.initial_tenant = initial_tenant or TenantId.generate()
         self.initial_timeline = initial_timeline or TimelineId.generate()
+        self.enable_generations = False
         self.scrub_on_exit = False
         self.test_output_dir = test_output_dir
 
@@ -740,6 +741,9 @@ class NeonEnvBuilder:
                 sk.stop(immediate=True)
             self.env.pageserver.stop(immediate=True)
 
+            if self.env.attachment_service is not None:
+                self.env.attachment_service.stop(immediate=True)
+
             cleanup_error = None
 
             if self.scrub_on_exit:
@@ -802,6 +806,8 @@ class NeonEnv:
         the tenant id
     """
 
+    PAGESERVER_ID = 1
+
     def __init__(self, config: NeonEnvBuilder):
         self.repo_dir = config.repo_dir
         self.rust_log_override = config.rust_log_override
@@ -825,6 +831,14 @@ class NeonEnv:
         self.initial_tenant = config.initial_tenant
         self.initial_timeline = config.initial_timeline
 
+        if config.enable_generations:
+            attachment_service_port = self.port_distributor.get_port()
+            self.control_plane_api: Optional[str] = f"http://127.0.0.1:{attachment_service_port}"
+            self.attachment_service: Optional[NeonAttachmentService] = NeonAttachmentService(self)
+        else:
+            self.control_plane_api = None
+            self.attachment_service = None
+
         # Create a config file corresponding to the options
         toml = textwrap.dedent(
             f"""
@@ -850,7 +864,7 @@ class NeonEnv:
         toml += textwrap.dedent(
             f"""
             [pageserver]
-            id=1
+            id={self.PAGESERVER_ID}
             listen_pg_addr = 'localhost:{pageserver_port.pg}'
             listen_http_addr = 'localhost:{pageserver_port.http}'
             pg_auth_type = '{pg_auth_type}'
@@ -858,6 +872,13 @@ class NeonEnv:
         """
         )
 
+        if self.control_plane_api is not None:
+            toml += textwrap.dedent(
+                f"""
+                control_plane_api = '{self.control_plane_api}'
+            """
+            )
+
         # Create a corresponding NeonPageserver object
         self.pageserver = NeonPageserver(
             self, port=pageserver_port, config_override=config.pageserver_config_override
@@ -904,6 +925,9 @@ class NeonEnv:
     def start(self):
         # Start up broker, pageserver and all safekeepers
         self.broker.try_start()
+
+        if self.attachment_service is not None:
+            self.attachment_service.start()
         self.pageserver.start()
 
         for safekeeper in self.safekeepers:
@@ -1331,6 +1355,16 @@ class NeonCli(AbstractNeonCli):
             res.check_returncode()
             return res
 
+    def attachment_service_start(self):
+        cmd = ["attachment_service", "start"]
+        return self.raw_cli(cmd)
+
+    def attachment_service_stop(self, immediate: bool):
+        cmd = ["attachment_service", "stop"]
+        if immediate:
+            cmd.extend(["-m", "immediate"])
+        return self.raw_cli(cmd)
+
     def pageserver_start(
         self,
         overrides: Tuple[str, ...] = (),
@@ -1512,6 +1546,35 @@ class ComputeCtl(AbstractNeonCli):
     COMMAND = "compute_ctl"
 
 
+class NeonAttachmentService:
+    def __init__(self, env: NeonEnv):
+        self.env = env
+        self.running = False
+
+    def start(self):
+        assert not self.running
+        self.env.neon_cli.attachment_service_start()
+        self.running = True
+        return self
+
+    def stop(self, immediate: bool = False) -> "NeonAttachmentService":
+        if self.running:
+            self.env.neon_cli.attachment_service_stop(immediate)
+            self.running = False
+        return self
+
+    def __enter__(self) -> "NeonAttachmentService":
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc: Optional[BaseException],
+        tb: Optional[TracebackType],
+    ):
+        self.stop(immediate=True)
+
+
 class NeonPageserver(PgProtocol):
     """
     An object representing a running pageserver.
@@ -1675,6 +1738,26 @@ class NeonPageserver(PgProtocol):
 
         return None
 
+    def tenant_attach(
+        self, tenant_id: TenantId, config: None | Dict[str, Any] = None, config_null: bool = False
+    ):
+        """
+        Tenant attachment passes through here to acquire a generation number before proceeding
+        to call into the pageserver HTTP client.
+        """
+        if self.env.attachment_service is not None:
+            response = requests.post(
+                f"{self.env.control_plane_api}/attach_hook",
+                json={"tenant_id": str(tenant_id), "pageserver_id": self.env.PAGESERVER_ID},
+            )
+            response.raise_for_status()
+            generation = response.json()["gen"]
+        else:
+            generation = None
+
+        client = self.env.pageserver.http_client()
+        return client.tenant_attach(tenant_id, config, config_null, generation=generation)
+
 
 def append_pageserver_param_overrides(
     params_to_update: List[str],
diff --git a/test_runner/fixtures/pageserver/http.py b/test_runner/fixtures/pageserver/http.py
index a179ebdd09..9373073abf 100644
--- a/test_runner/fixtures/pageserver/http.py
+++ b/test_runner/fixtures/pageserver/http.py
@@ -186,18 +186,25 @@ class PageserverHttpClient(requests.Session):
         return TenantId(new_tenant_id)
 
     def tenant_attach(
-        self, tenant_id: TenantId, config: None | Dict[str, Any] = None, config_null: bool = False
+        self,
+        tenant_id: TenantId,
+        config: None | Dict[str, Any] = None,
+        config_null: bool = False,
+        generation: Optional[int] = None,
     ):
         if config_null:
             assert config is None
-            body = "null"
+            body: Any = None
         else:
             # null-config is prohibited by the API
             config = config or {}
-            body = json.dumps({"config": config})
+            body = {"config": config}
+            if generation is not None:
+                body.update({"generation": generation})
+
         res = self.post(
             f"http://localhost:{self.port}/v1/tenant/{tenant_id}/attach",
-            data=body,
+            data=json.dumps(body),
             headers={"Content-Type": "application/json"},
         )
         self.verbose_error(res)
diff --git a/test_runner/regress/test_pageserver_restart.py b/test_runner/regress/test_pageserver_restart.py
index 567200ba1f..ad625df1cc 100644
--- a/test_runner/regress/test_pageserver_restart.py
+++ b/test_runner/regress/test_pageserver_restart.py
@@ -8,7 +8,9 @@ from fixtures.remote_storage import s3_storage
 
 # Test restarting page server, while safekeeper and compute node keep
 # running.
-def test_pageserver_restart(neon_env_builder: NeonEnvBuilder):
+@pytest.mark.parametrize("generations", [True, False])
+def test_pageserver_restart(neon_env_builder: NeonEnvBuilder, generations: bool):
+    neon_env_builder.enable_generations = generations
     neon_env_builder.enable_remote_storage(remote_storage_kind=s3_storage())
     neon_env_builder.enable_scrub_on_exit()
 
diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py
index 8b75d35200..3d9655cb87 100644
--- a/test_runner/regress/test_remote_storage.py
+++ b/test_runner/regress/test_remote_storage.py
@@ -52,9 +52,9 @@ from requests import ReadTimeout
 #
 # The tests are done for all types of remote storage pageserver supports.
 @pytest.mark.parametrize("remote_storage_kind", available_remote_storages())
+@pytest.mark.parametrize("generations", [True, False])
 def test_remote_storage_backup_and_restore(
-    neon_env_builder: NeonEnvBuilder,
-    remote_storage_kind: RemoteStorageKind,
+    neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind, generations: bool
 ):
     # Use this test to check more realistic SK ids: some etcd key parsing bugs were related,
     # and this test needs SK to write data to pageserver, so it will be visible
@@ -64,6 +64,8 @@ def test_remote_storage_backup_and_restore(
         remote_storage_kind=remote_storage_kind,
     )
 
+    neon_env_builder.enable_generations = generations
+
     # Exercise retry code path by making all uploads and downloads fail for the
     # first time. The retries print INFO-messages to the log; we will check
     # that they are present after the test.
@@ -154,7 +156,7 @@ def test_remote_storage_backup_and_restore(
     # background task to load the tenant. In that background task,
     # listing the remote timelines will fail because of the failpoint,
     # and the tenant will be marked as Broken.
-    client.tenant_attach(tenant_id)
+    env.pageserver.tenant_attach(tenant_id)
 
     tenant_info = wait_until_tenant_state(pageserver_http, tenant_id, "Broken", 15)
     assert tenant_info["attachment_status"] == {
@@ -164,7 +166,7 @@ def test_remote_storage_backup_and_restore(
 
     # Ensure that even though the tenant is broken, we can't attach it again.
     with pytest.raises(Exception, match=f"tenant {tenant_id} already exists, state: Broken"):
-        client.tenant_attach(tenant_id)
+        env.pageserver.tenant_attach(tenant_id)
 
     # Restart again, this implicitly clears the failpoint.
     # test_remote_failures=1 remains active, though, as it's in the pageserver config.
@@ -182,7 +184,7 @@ def test_remote_storage_backup_and_restore(
     # Ensure that the pageserver remembers that the tenant was attaching, by
     # trying to attach it again. It should fail.
     with pytest.raises(Exception, match=f"tenant {tenant_id} already exists, state:"):
-        client.tenant_attach(tenant_id)
+        env.pageserver.tenant_attach(tenant_id)
     log.info("waiting for tenant to become active. this should be quick with on-demand download")
 
     wait_until_tenant_active(
@@ -362,7 +364,7 @@ def test_remote_storage_upload_queue_retries(
     env.pageserver.start()
     client = env.pageserver.http_client()
 
-    client.tenant_attach(tenant_id)
+    env.pageserver.tenant_attach(tenant_id)
 
     wait_until_tenant_active(client, tenant_id)
 
@@ -499,7 +501,7 @@ def test_remote_timeline_client_calls_started_metric(
     env.pageserver.start()
     client = env.pageserver.http_client()
 
-    client.tenant_attach(tenant_id)
+    env.pageserver.tenant_attach(tenant_id)
 
     wait_until_tenant_active(client, tenant_id)
 

From d5f1858f7877fc149f5b4ec00ac12315cfddf3a9 Mon Sep 17 00:00:00 2001
From: Alexander Bayandin <alexander@neon.tech>
Date: Wed, 6 Sep 2023 16:50:59 +0100
Subject: [PATCH 13/61] approved-for-ci-run.yml: use different tokens (#5218)

## Problem

`CI_ACCESS_TOKEN` has quite limited access (which is good), but this
doesn't allow it to remove labels from PRs (which is bad)

## Summary of changes
- Use `GITHUB_TOKEN` to remove labels
- Use `CI_ACCESS_TOKEN` to create PRs
---
 .github/workflows/approved-for-ci-run.yml | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/approved-for-ci-run.yml b/.github/workflows/approved-for-ci-run.yml
index 256ab8b438..81955a602e 100644
--- a/.github/workflows/approved-for-ci-run.yml
+++ b/.github/workflows/approved-for-ci-run.yml
@@ -17,9 +17,11 @@ on:
       - labeled
 
 env:
-  GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
+  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   PR_NUMBER: ${{ github.event.pull_request.number }}
 
+permissions: write-all
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
 
@@ -60,14 +62,17 @@ jobs:
       - run: git push --force origin "ci-run/pr-${PR_NUMBER}"
 
       - name: Create a Pull Request for CI run (if required)
+        env:
+          GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
         run: |
           HEAD="ci-run/pr-${PR_NUMBER}"
+          BODY="This Pull Request was create automatically to run CI pipeline for #${PR_NUMBER}.\n\nPlease do not alter or merge/close it.\n\nFeel free to comment the original PR."
 
-          ALREADY_CREATED=$(gh pr --repo "${GITHUB_REPOSITORY}" list --head "${HEAD}" --base main --json number --jq '.[].number')
+          ALREADY_CREATED=$(gh pr --repo "${GITHUB_REPOSITORY}" list --head "${HEAD}" --base "main" --json "number" --jq '.[].number')
           if [ -z "${ALREADY_CREATED}" ]; then
-            gh pr --repo "${GITHUB_REPOSITORY}" create  --title "[DO NOT MERGE] CI run for PR #${PR_NUMBER}" \
-                                                        --body "Ref #${PR_NUMBER}" \
+            gh pr --repo "${GITHUB_REPOSITORY}" create  --title "CI run for PR #${PR_NUMBER}" \
+                                                        --body "${BODY}" \
                                                         --head "${HEAD}" \
-                                                        --base main \
+                                                        --base "main" \
                                                         --draft
           fi

From 5e00c441692ff4746a6604583c04771736f0ee6b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arpad=20M=C3=BCller?= <arpad-m@users.noreply.github.com>
Date: Wed, 6 Sep 2023 18:17:12 +0200
Subject: [PATCH 14/61] Add WriteBlobWriter buffering and make
 VirtualFile::{write,write_all} async (#5203)

## Problem

We want to convert the `VirtualFile` APIs to async fn so that we can
adopt one of the async I/O solutions.

## Summary of changes

This PR is a follow-up of #5189, #5190, and #5195, and does the
following:

* Move the used `Write` trait functions of `VirtualFile` into inherent
functions
* Add optional buffering to `WriteBlobWriter`. The buffer is discarded
on drop, similarly to how tokio's `BufWriter` does it: drop is neither
async nor does it support errors.
* Remove the generics by `Write` impl of `WriteBlobWriter`, alwaays
using `VirtualFile`
* Rename `WriteBlobWriter` to `BlobWriter`
* Make various functions in the write path async, like
`VirtualFile::{write,write_all}`.

Part of #4743.
---
 pageserver/src/tenant/blob_io.rs              | 250 ++++++++++++++++--
 pageserver/src/tenant/block_io.rs             |   4 +
 .../src/tenant/storage_layer/delta_layer.rs   |  26 +-
 .../src/tenant/storage_layer/image_layer.rs   |  11 +-
 pageserver/src/virtual_file.rs                |  96 ++++---
 5 files changed, 303 insertions(+), 84 deletions(-)

diff --git a/pageserver/src/tenant/blob_io.rs b/pageserver/src/tenant/blob_io.rs
index ba3542d869..71db8d2978 100644
--- a/pageserver/src/tenant/blob_io.rs
+++ b/pageserver/src/tenant/blob_io.rs
@@ -13,6 +13,7 @@
 //!
 use crate::page_cache::PAGE_SZ;
 use crate::tenant::block_io::BlockCursor;
+use crate::virtual_file::VirtualFile;
 use std::cmp::min;
 use std::io::{Error, ErrorKind};
 
@@ -83,20 +84,24 @@ impl<'a> BlockCursor<'a> {
     }
 }
 
+/// A wrapper of `VirtualFile` that allows users to write blobs.
 ///
-/// An implementation of BlobWriter to write blobs to anything that
-/// implements std::io::Write.
-///
-pub struct WriteBlobWriter<W> {
-    inner: W,
+/// If a `BlobWriter` is dropped, the internal buffer will be
+/// discarded. You need to call [`flush_buffer`](Self::flush_buffer)
+/// manually before dropping.
+pub struct BlobWriter<const BUFFERED: bool> {
+    inner: VirtualFile,
     offset: u64,
+    /// A buffer to save on write calls, only used if BUFFERED=true
+    buf: Vec<u8>,
 }
 
-impl<W> WriteBlobWriter<W> {
-    pub fn new(inner: W, start_offset: u64) -> Self {
-        WriteBlobWriter {
+impl<const BUFFERED: bool> BlobWriter<BUFFERED> {
+    pub fn new(inner: VirtualFile, start_offset: u64) -> Self {
+        Self {
             inner,
             offset: start_offset,
+            buf: Vec::with_capacity(Self::CAPACITY),
         }
     }
 
@@ -104,20 +109,70 @@ impl<W> WriteBlobWriter<W> {
         self.offset
     }
 
-    /// Access the underlying Write object.
-    ///
-    /// NOTE: WriteBlobWriter keeps track of the current write offset. If
-    /// you write something directly to the inner Write object, it makes the
-    /// internally tracked 'offset' to go out of sync. So don't do that.
-    pub fn into_inner(self) -> W {
-        self.inner
-    }
-}
+    const CAPACITY: usize = if BUFFERED { PAGE_SZ } else { 0 };
+
+    #[inline(always)]
+    /// Writes the given buffer directly to the underlying `VirtualFile`.
+    /// You need to make sure that the internal buffer is empty, otherwise
+    /// data will be written in wrong order.
+    async fn write_all_unbuffered(&mut self, src_buf: &[u8]) -> Result<(), Error> {
+        self.inner.write_all(src_buf).await?;
+        self.offset += src_buf.len() as u64;
+        Ok(())
+    }
+
+    #[inline(always)]
+    /// Flushes the internal buffer to the underlying `VirtualFile`.
+    pub async fn flush_buffer(&mut self) -> Result<(), Error> {
+        self.inner.write_all(&self.buf).await?;
+        self.buf.clear();
+        Ok(())
+    }
+
+    #[inline(always)]
+    /// Writes as much of `src_buf` into the internal buffer as it fits
+    fn write_into_buffer(&mut self, src_buf: &[u8]) -> usize {
+        let remaining = Self::CAPACITY - self.buf.len();
+        let to_copy = src_buf.len().min(remaining);
+        self.buf.extend_from_slice(&src_buf[..to_copy]);
+        self.offset += to_copy as u64;
+        to_copy
+    }
+
+    /// Internal, possibly buffered, write function
+    async fn write_all(&mut self, mut src_buf: &[u8]) -> Result<(), Error> {
+        if !BUFFERED {
+            assert!(self.buf.is_empty());
+            self.write_all_unbuffered(src_buf).await?;
+            return Ok(());
+        }
+        let remaining = Self::CAPACITY - self.buf.len();
+        // First try to copy as much as we can into the buffer
+        if remaining > 0 {
+            let copied = self.write_into_buffer(src_buf);
+            src_buf = &src_buf[copied..];
+        }
+        // Then, if the buffer is full, flush it out
+        if self.buf.len() == Self::CAPACITY {
+            self.flush_buffer().await?;
+        }
+        // Finally, write the tail of src_buf:
+        // If it wholly fits into the buffer without
+        // completely filling it, then put it there.
+        // If not, write it out directly.
+        if !src_buf.is_empty() {
+            assert_eq!(self.buf.len(), 0);
+            if src_buf.len() < Self::CAPACITY {
+                let copied = self.write_into_buffer(src_buf);
+                // We just verified above that src_buf fits into our internal buffer.
+                assert_eq!(copied, src_buf.len());
+            } else {
+                self.write_all_unbuffered(src_buf).await?;
+            }
+        }
+        Ok(())
+    }
 
-impl<W> WriteBlobWriter<W>
-where
-    W: std::io::Write,
-{
     /// Write a blob of data. Returns the offset that it was written to,
     /// which can be used to retrieve the data later.
     pub async fn write_blob(&mut self, srcbuf: &[u8]) -> Result<u64, Error> {
@@ -126,8 +181,7 @@ where
         if srcbuf.len() < 128 {
             // Short blob. Write a 1-byte length header
             let len_buf = srcbuf.len() as u8;
-            self.inner.write_all(&[len_buf])?;
-            self.offset += 1;
+            self.write_all(&[len_buf]).await?;
         } else {
             // Write a 4-byte length header
             if srcbuf.len() > 0x7fff_ffff {
@@ -138,11 +192,153 @@ where
             }
             let mut len_buf = ((srcbuf.len()) as u32).to_be_bytes();
             len_buf[0] |= 0x80;
-            self.inner.write_all(&len_buf)?;
-            self.offset += 4;
+            self.write_all(&len_buf).await?;
         }
-        self.inner.write_all(srcbuf)?;
-        self.offset += srcbuf.len() as u64;
+        self.write_all(srcbuf).await?;
         Ok(offset)
     }
 }
+
+impl BlobWriter<true> {
+    /// Access the underlying `VirtualFile`.
+    ///
+    /// This function flushes the internal buffer before giving access
+    /// to the underlying `VirtualFile`.
+    pub async fn into_inner(mut self) -> Result<VirtualFile, Error> {
+        self.flush_buffer().await?;
+        Ok(self.inner)
+    }
+
+    /// Access the underlying `VirtualFile`.
+    ///
+    /// Unlike [`into_inner`](Self::into_inner), this doesn't flush
+    /// the internal buffer before giving access.
+    pub fn into_inner_no_flush(self) -> VirtualFile {
+        self.inner
+    }
+}
+
+impl BlobWriter<false> {
+    /// Access the underlying `VirtualFile`.
+    pub fn into_inner(self) -> VirtualFile {
+        self.inner
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::tenant::block_io::BlockReaderRef;
+    use rand::{Rng, SeedableRng};
+
+    async fn round_trip_test<const BUFFERED: bool>(blobs: &[Vec<u8>]) -> Result<(), Error> {
+        let temp_dir = tempfile::tempdir()?;
+        let path = temp_dir.path().join("file");
+
+        // Write part (in block to drop the file)
+        let mut offsets = Vec::new();
+        {
+            let file = VirtualFile::create(&path)?;
+            let mut wtr = BlobWriter::<BUFFERED>::new(file, 0);
+            for blob in blobs.iter() {
+                let offs = wtr.write_blob(blob).await?;
+                offsets.push(offs);
+            }
+            // Write out one page worth of zeros so that we can
+            // read again with read_blk
+            let offs = wtr.write_blob(&vec![0; PAGE_SZ]).await?;
+            println!("Writing final blob at offs={offs}");
+            wtr.flush_buffer().await?;
+        }
+
+        let file = VirtualFile::open(&path)?;
+        let rdr = BlockReaderRef::VirtualFile(&file);
+        let rdr = BlockCursor::new(rdr);
+        for (idx, (blob, offset)) in blobs.iter().zip(offsets.iter()).enumerate() {
+            let blob_read = rdr.read_blob(*offset).await?;
+            assert_eq!(
+                blob, &blob_read,
+                "mismatch for idx={idx} at offset={offset}"
+            );
+        }
+        Ok(())
+    }
+
+    fn random_array(len: usize) -> Vec<u8> {
+        let mut rng = rand::thread_rng();
+        (0..len).map(|_| rng.gen()).collect::<_>()
+    }
+
+    #[tokio::test]
+    async fn test_one() -> Result<(), Error> {
+        let blobs = &[vec![12, 21, 22]];
+        round_trip_test::<false>(blobs).await?;
+        round_trip_test::<true>(blobs).await?;
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_hello_simple() -> Result<(), Error> {
+        let blobs = &[
+            vec![0, 1, 2, 3],
+            b"Hello, World!".to_vec(),
+            Vec::new(),
+            b"foobar".to_vec(),
+        ];
+        round_trip_test::<false>(blobs).await?;
+        round_trip_test::<true>(blobs).await?;
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_really_big_array() -> Result<(), Error> {
+        let blobs = &[
+            b"test".to_vec(),
+            random_array(10 * PAGE_SZ),
+            b"foobar".to_vec(),
+        ];
+        round_trip_test::<false>(blobs).await?;
+        round_trip_test::<true>(blobs).await?;
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_arrays_inc() -> Result<(), Error> {
+        let blobs = (0..PAGE_SZ / 8)
+            .map(|v| random_array(v * 16))
+            .collect::<Vec<_>>();
+        round_trip_test::<false>(&blobs).await?;
+        round_trip_test::<true>(&blobs).await?;
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_arrays_random_size() -> Result<(), Error> {
+        let mut rng = rand::rngs::StdRng::seed_from_u64(42);
+        let blobs = (0..1024)
+            .map(|_| {
+                let mut sz: u16 = rng.gen();
+                // Make 50% of the arrays small
+                if rng.gen() {
+                    sz |= 63;
+                }
+                random_array(sz.into())
+            })
+            .collect::<Vec<_>>();
+        round_trip_test::<false>(&blobs).await?;
+        round_trip_test::<true>(&blobs).await?;
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_arrays_page_boundary() -> Result<(), Error> {
+        let blobs = &[
+            random_array(PAGE_SZ - 4),
+            random_array(PAGE_SZ - 4),
+            random_array(PAGE_SZ - 4),
+        ];
+        round_trip_test::<false>(blobs).await?;
+        round_trip_test::<true>(blobs).await?;
+        Ok(())
+    }
+}
diff --git a/pageserver/src/tenant/block_io.rs b/pageserver/src/tenant/block_io.rs
index 645ec81036..abb636579d 100644
--- a/pageserver/src/tenant/block_io.rs
+++ b/pageserver/src/tenant/block_io.rs
@@ -76,6 +76,8 @@ pub(crate) enum BlockReaderRef<'a> {
     Adapter(Adapter<&'a DeltaLayerInner>),
     #[cfg(test)]
     TestDisk(&'a super::disk_btree::tests::TestDisk),
+    #[cfg(test)]
+    VirtualFile(&'a VirtualFile),
 }
 
 impl<'a> BlockReaderRef<'a> {
@@ -88,6 +90,8 @@ impl<'a> BlockReaderRef<'a> {
             Adapter(r) => r.read_blk(blknum).await,
             #[cfg(test)]
             TestDisk(r) => r.read_blk(blknum),
+            #[cfg(test)]
+            VirtualFile(r) => r.read_blk(blknum).await,
         }
     }
 }
diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs
index e95c422d29..b6fbf98962 100644
--- a/pageserver/src/tenant/storage_layer/delta_layer.rs
+++ b/pageserver/src/tenant/storage_layer/delta_layer.rs
@@ -31,7 +31,7 @@ use crate::config::PageServerConf;
 use crate::context::RequestContext;
 use crate::page_cache::PAGE_SZ;
 use crate::repository::{Key, Value, KEY_SIZE};
-use crate::tenant::blob_io::WriteBlobWriter;
+use crate::tenant::blob_io::BlobWriter;
 use crate::tenant::block_io::{BlockBuf, BlockCursor, BlockLease, BlockReader, FileBlockReader};
 use crate::tenant::disk_btree::{DiskBtreeBuilder, DiskBtreeReader, VisitDirection};
 use crate::tenant::storage_layer::{
@@ -46,7 +46,6 @@ use rand::{distributions::Alphanumeric, Rng};
 use serde::{Deserialize, Serialize};
 use std::fs::{self, File};
 use std::io::SeekFrom;
-use std::io::{BufWriter, Write};
 use std::ops::Range;
 use std::os::unix::fs::FileExt;
 use std::path::{Path, PathBuf};
@@ -583,7 +582,7 @@ struct DeltaLayerWriterInner {
 
     tree: DiskBtreeBuilder<BlockBuf, DELTA_KEY_SIZE>,
 
-    blob_writer: WriteBlobWriter<BufWriter<VirtualFile>>,
+    blob_writer: BlobWriter<true>,
 }
 
 impl DeltaLayerWriterInner {
@@ -608,8 +607,7 @@ impl DeltaLayerWriterInner {
         let mut file = VirtualFile::create(&path)?;
         // make room for the header block
         file.seek(SeekFrom::Start(PAGE_SZ as u64)).await?;
-        let buf_writer = BufWriter::new(file);
-        let blob_writer = WriteBlobWriter::new(buf_writer, PAGE_SZ as u64);
+        let blob_writer = BlobWriter::new(file, PAGE_SZ as u64);
 
         // Initialize the b-tree index builder
         let block_buf = BlockBuf::new();
@@ -667,15 +665,14 @@ impl DeltaLayerWriterInner {
         let index_start_blk =
             ((self.blob_writer.size() + PAGE_SZ as u64 - 1) / PAGE_SZ as u64) as u32;
 
-        let buf_writer = self.blob_writer.into_inner();
-        let mut file = buf_writer.into_inner()?;
+        let mut file = self.blob_writer.into_inner().await?;
 
         // Write out the index
         let (index_root_blk, block_buf) = self.tree.finish()?;
         file.seek(SeekFrom::Start(index_start_blk as u64 * PAGE_SZ as u64))
             .await?;
         for buf in block_buf.blocks {
-            file.write_all(buf.as_ref())?;
+            file.write_all(buf.as_ref()).await?;
         }
         assert!(self.lsn_range.start < self.lsn_range.end);
         // Fill in the summary on blk 0
@@ -700,7 +697,7 @@ impl DeltaLayerWriterInner {
             );
         }
         file.seek(SeekFrom::Start(0)).await?;
-        file.write_all(&buf)?;
+        file.write_all(&buf).await?;
 
         let metadata = file
             .metadata()
@@ -840,13 +837,10 @@ impl DeltaLayerWriter {
 impl Drop for DeltaLayerWriter {
     fn drop(&mut self) {
         if let Some(inner) = self.inner.take() {
-            match inner.blob_writer.into_inner().into_inner() {
-                Ok(vfile) => vfile.remove(),
-                Err(err) => warn!(
-                    "error while flushing buffer of image layer temporary file: {}",
-                    err
-                ),
-            }
+            // We want to remove the virtual file here, so it's fine to not
+            // having completely flushed unwritten data.
+            let vfile = inner.blob_writer.into_inner_no_flush();
+            vfile.remove();
         }
     }
 }
diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs
index 62bfeb8eec..8f7fb8175c 100644
--- a/pageserver/src/tenant/storage_layer/image_layer.rs
+++ b/pageserver/src/tenant/storage_layer/image_layer.rs
@@ -27,7 +27,7 @@ use crate::config::PageServerConf;
 use crate::context::RequestContext;
 use crate::page_cache::PAGE_SZ;
 use crate::repository::{Key, KEY_SIZE};
-use crate::tenant::blob_io::WriteBlobWriter;
+use crate::tenant::blob_io::BlobWriter;
 use crate::tenant::block_io::{BlockBuf, BlockReader, FileBlockReader};
 use crate::tenant::disk_btree::{DiskBtreeBuilder, DiskBtreeReader, VisitDirection};
 use crate::tenant::storage_layer::{
@@ -43,7 +43,6 @@ use rand::{distributions::Alphanumeric, Rng};
 use serde::{Deserialize, Serialize};
 use std::fs::{self, File};
 use std::io::SeekFrom;
-use std::io::Write;
 use std::ops::Range;
 use std::os::unix::prelude::FileExt;
 use std::path::{Path, PathBuf};
@@ -511,7 +510,7 @@ struct ImageLayerWriterInner {
     key_range: Range<Key>,
     lsn: Lsn,
 
-    blob_writer: WriteBlobWriter<VirtualFile>,
+    blob_writer: BlobWriter<false>,
     tree: DiskBtreeBuilder<BlockBuf, KEY_SIZE>,
 }
 
@@ -544,7 +543,7 @@ impl ImageLayerWriterInner {
         )?;
         // make room for the header block
         file.seek(SeekFrom::Start(PAGE_SZ as u64)).await?;
-        let blob_writer = WriteBlobWriter::new(file, PAGE_SZ as u64);
+        let blob_writer = BlobWriter::new(file, PAGE_SZ as u64);
 
         // Initialize the b-tree index builder
         let block_buf = BlockBuf::new();
@@ -594,7 +593,7 @@ impl ImageLayerWriterInner {
             .await?;
         let (index_root_blk, block_buf) = self.tree.finish()?;
         for buf in block_buf.blocks {
-            file.write_all(buf.as_ref())?;
+            file.write_all(buf.as_ref()).await?;
         }
 
         // Fill in the summary on blk 0
@@ -619,7 +618,7 @@ impl ImageLayerWriterInner {
             );
         }
         file.seek(SeekFrom::Start(0)).await?;
-        file.write_all(&buf)?;
+        file.write_all(&buf).await?;
 
         let metadata = file
             .metadata()
diff --git a/pageserver/src/virtual_file.rs b/pageserver/src/virtual_file.rs
index 91430ef6eb..c4241c4270 100644
--- a/pageserver/src/virtual_file.rs
+++ b/pageserver/src/virtual_file.rs
@@ -14,7 +14,7 @@ use crate::metrics::{STORAGE_IO_SIZE, STORAGE_IO_TIME};
 use crate::tenant::TENANTS_SEGMENT_NAME;
 use once_cell::sync::OnceCell;
 use std::fs::{self, File, OpenOptions};
-use std::io::{Error, ErrorKind, Seek, SeekFrom, Write};
+use std::io::{Error, ErrorKind, Seek, SeekFrom};
 use std::os::unix::fs::FileExt;
 use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
@@ -301,6 +301,7 @@ impl VirtualFile {
         )
         .map_err(CrashsafeOverwriteError::CreateTempfile)?;
         file.write_all(content)
+            .await
             .map_err(CrashsafeOverwriteError::WriteContents)?;
         file.sync_all()
             .map_err(CrashsafeOverwriteError::SyncTempfile)?;
@@ -433,22 +434,6 @@ impl VirtualFile {
         Ok(self.pos)
     }
 
-    #[cfg(test)]
-    async fn read_to_end(&mut self, buf: &mut Vec<u8>) -> Result<(), Error> {
-        loop {
-            let mut tmp = [0; 128];
-            match self.read_at(&mut tmp, self.pos).await {
-                Ok(0) => return Ok(()),
-                Ok(n) => {
-                    self.pos += n as u64;
-                    buf.extend_from_slice(&tmp[..n]);
-                }
-                Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => {}
-                Err(e) => return Err(e),
-            }
-        }
-    }
-
     // Copied from https://doc.rust-lang.org/1.72.0/src/std/os/unix/fs.rs.html#117-135
     pub async fn read_exact_at(&self, mut buf: &mut [u8], mut offset: u64) -> Result<(), Error> {
         while !buf.is_empty() {
@@ -473,7 +458,7 @@ impl VirtualFile {
     // Copied from https://doc.rust-lang.org/1.72.0/src/std/os/unix/fs.rs.html#219-235
     pub async fn write_all_at(&self, mut buf: &[u8], mut offset: u64) -> Result<(), Error> {
         while !buf.is_empty() {
-            match self.write_at(buf, offset) {
+            match self.write_at(buf, offset).await {
                 Ok(0) => {
                     return Err(Error::new(
                         std::io::ErrorKind::WriteZero,
@@ -491,6 +476,32 @@ impl VirtualFile {
         Ok(())
     }
 
+    pub async fn write_all(&mut self, mut buf: &[u8]) -> Result<(), Error> {
+        while !buf.is_empty() {
+            match self.write(buf).await {
+                Ok(0) => {
+                    return Err(Error::new(
+                        std::io::ErrorKind::WriteZero,
+                        "failed to write whole buffer",
+                    ));
+                }
+                Ok(n) => {
+                    buf = &buf[n..];
+                }
+                Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => {}
+                Err(e) => return Err(e),
+            }
+        }
+        Ok(())
+    }
+
+    async fn write(&mut self, buf: &[u8]) -> Result<usize, std::io::Error> {
+        let pos = self.pos;
+        let n = self.write_at(buf, pos).await?;
+        self.pos += n as u64;
+        Ok(n)
+    }
+
     pub async fn read_at(&self, buf: &mut [u8], offset: u64) -> Result<usize, Error> {
         let result = self.with_file("read", |file| file.read_at(buf, offset))?;
         if let Ok(size) = result {
@@ -501,7 +512,7 @@ impl VirtualFile {
         result
     }
 
-    pub fn write_at(&self, buf: &[u8], offset: u64) -> Result<usize, Error> {
+    async fn write_at(&self, buf: &[u8], offset: u64) -> Result<usize, Error> {
         let result = self.with_file("write", |file| file.write_at(buf, offset))?;
         if let Ok(size) = result {
             STORAGE_IO_SIZE
@@ -512,6 +523,35 @@ impl VirtualFile {
     }
 }
 
+#[cfg(test)]
+impl VirtualFile {
+    pub(crate) async fn read_blk(
+        &self,
+        blknum: u32,
+    ) -> Result<crate::tenant::block_io::BlockLease<'_>, std::io::Error> {
+        use crate::page_cache::PAGE_SZ;
+        let mut buf = [0; PAGE_SZ];
+        self.read_exact_at(&mut buf, blknum as u64 * (PAGE_SZ as u64))
+            .await?;
+        Ok(std::sync::Arc::new(buf).into())
+    }
+
+    async fn read_to_end(&mut self, buf: &mut Vec<u8>) -> Result<(), Error> {
+        loop {
+            let mut tmp = [0; 128];
+            match self.read_at(&mut tmp, self.pos).await {
+                Ok(0) => return Ok(()),
+                Ok(n) => {
+                    self.pos += n as u64;
+                    buf.extend_from_slice(&tmp[..n]);
+                }
+                Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => {}
+                Err(e) => return Err(e),
+            }
+        }
+    }
+}
+
 impl Drop for VirtualFile {
     /// If a VirtualFile is dropped, close the underlying file if it was open.
     fn drop(&mut self) {
@@ -532,21 +572,6 @@ impl Drop for VirtualFile {
     }
 }
 
-impl Write for VirtualFile {
-    fn write(&mut self, buf: &[u8]) -> Result<usize, std::io::Error> {
-        let pos = self.pos;
-        let n = self.write_at(buf, pos)?;
-        self.pos += n as u64;
-        Ok(n)
-    }
-
-    fn flush(&mut self) -> Result<(), std::io::Error> {
-        // flush is no-op for File (at least on unix), so we don't need to do
-        // anything here either.
-        Ok(())
-    }
-}
-
 impl OpenFiles {
     fn new(num_slots: usize) -> OpenFiles {
         let mut slots = Box::new(Vec::with_capacity(num_slots));
@@ -600,6 +625,7 @@ mod tests {
     use rand::seq::SliceRandom;
     use rand::thread_rng;
     use rand::Rng;
+    use std::io::Write;
     use std::sync::Arc;
 
     enum MaybeVirtualFile {
@@ -628,7 +654,7 @@ mod tests {
         }
         async fn write_all(&mut self, buf: &[u8]) -> Result<(), Error> {
             match self {
-                MaybeVirtualFile::VirtualFile(file) => file.write_all(buf),
+                MaybeVirtualFile::VirtualFile(file) => file.write_all(buf).await,
                 MaybeVirtualFile::File(file) => file.write_all(buf),
             }
         }

From 76a96b07457b70a7f70ec3bca534a94f7d4474ff Mon Sep 17 00:00:00 2001
From: Alexander Bayandin <alexander@neon.tech>
Date: Wed, 6 Sep 2023 17:52:21 +0100
Subject: [PATCH 15/61] Notify Slack channel about upcoming releases (#5197)

## Problem

When the next release is coming, we want to let everyone know about it by
posting a message to the Slack channel with a list of commits.

## Summary of changes
- `.github/workflows/release-notify.yml` is added
- the workflow sends a message to
`vars.SLACK_UPCOMING_RELEASE_CHANNEL_ID` (or
[#test-release-notifications](https://neondb.slack.com/archives/C05QQ9J1BRC)
if not configured)
- On each PR update, the workflow updates the list of commits in the
message (it doesn't send additional messages)
---
 .github/workflows/release-notify.yml | 29 ++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 .github/workflows/release-notify.yml

diff --git a/.github/workflows/release-notify.yml b/.github/workflows/release-notify.yml
new file mode 100644
index 0000000000..ba396dba74
--- /dev/null
+++ b/.github/workflows/release-notify.yml
@@ -0,0 +1,29 @@
+name: Notify Slack channel about upcoming release
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.number }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    branches:
+      - release
+    types:
+      # Default types that triggers a workflow:
+      # - https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request
+      - opened
+      - synchronize
+      - reopened
+      # Additional types that we want to handle:
+      - closed
+
+jobs:
+  notify:
+    runs-on: [ ubuntu-latest ]
+
+    steps:
+      - uses: neondatabase/dev-actions/release-pr-notify@main
+        with:
+          slack-token: ${{ secrets.SLACK_BOT_TOKEN }}
+          slack-channel-id: ${{ vars.SLACK_UPCOMING_RELEASE_CHANNEL_ID || 'C05QQ9J1BRC' }} # if not set, then `#test-release-notifications`
+          github-token: ${{ secrets.GITHUB_TOKEN }}

From e4b1d6b30a699bed9e41d4cc41a68eaa9731c34a Mon Sep 17 00:00:00 2001
From: Alexander Bayandin <alexander@neon.tech>
Date: Wed, 6 Sep 2023 18:11:44 +0100
Subject: [PATCH 16/61] Misc post-merge fixes (#5219)

## Problem
- `SCALE: unbound variable` from
https://github.com/neondatabase/neon/pull/5079
- The layout of the GitHub auto-comment is broken if the code coverage
section follows flaky test section from
https://github.com/neondatabase/neon/pull/4999

## Summary of changes
- `benchmarking.yml`: Rename `SCALE` to `TEST_OLAP_SCALE`
- `comment-test-report.js`: Add an extra new-line before Code coverage
section
---
 .github/workflows/benchmarking.yml | 2 +-
 scripts/comment-test-report.js     | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml
index bea88e67a5..464b764d1a 100644
--- a/.github/workflows/benchmarking.yml
+++ b/.github/workflows/benchmarking.yml
@@ -476,7 +476,7 @@ jobs:
             ;;
         esac
 
-        CONNSTR_SECRET_NAME="BENCHMARK_${ENV_PLATFORM}_S${SCALE}_CONNSTR"
+        CONNSTR_SECRET_NAME="BENCHMARK_${ENV_PLATFORM}_S${TEST_OLAP_SCALE}_CONNSTR"
         echo "CONNSTR_SECRET_NAME=${CONNSTR_SECRET_NAME}" >> $GITHUB_ENV
 
     - name: Set up Connection String
diff --git a/scripts/comment-test-report.js b/scripts/comment-test-report.js
index f62d923dc4..055e3c1b2e 100755
--- a/scripts/comment-test-report.js
+++ b/scripts/comment-test-report.js
@@ -139,7 +139,7 @@ const reportSummary = async (params) => {
     // Print test resuls from the newest to the oldest Postgres version for release and debug builds.
     for (const pgVersion of Array.from(pgVersions).sort().reverse()) {
         if (Object.keys(failedTests[pgVersion]).length > 0) {
-            summary += `#### Failures on Posgres ${pgVersion}\n\n`
+            summary += `#### Failures on Postgres ${pgVersion}\n\n`
             for (const [testName, tests] of Object.entries(failedTests[pgVersion])) {
                 const links = []
                 for (const test of tests) {
@@ -188,7 +188,7 @@ const reportSummary = async (params) => {
 }
 
 const parseCoverageSummary = async ({ summaryJsonUrl, coverageUrl, fetch }) => {
-    let summary = `### Code coverage [full report](${coverageUrl})\n`
+    let summary = `\n### Code coverage ([full report](${coverageUrl}))\n`
 
     const coverage = await (await fetch(summaryJsonUrl)).json()
     for (const covType of Object.keys(coverage).sort()) {

From dfe2e5159aa66590603e8a636b6cffb9ada73b39 Mon Sep 17 00:00:00 2001
From: Chengpeng Yan <41809508+Reminiscent@users.noreply.github.com>
Date: Thu, 7 Sep 2023 01:57:03 +0800
Subject: [PATCH 17/61] remove the duplicate entries in postgresql.conf (#5090)

---
 compute_tools/src/config.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/compute_tools/src/config.rs b/compute_tools/src/config.rs
index 2da671a149..bc48a2110d 100644
--- a/compute_tools/src/config.rs
+++ b/compute_tools/src/config.rs
@@ -46,8 +46,6 @@ pub fn write_postgres_conf(
         writeln!(file, "{}", conf)?;
     }
 
-    write!(file, "{}", &spec.cluster.settings.as_pg_settings())?;
-
     // Add options for connecting to storage
     writeln!(file, "# Neon storage settings")?;
     if let Some(s) = &spec.pageserver_connstring {

From 9a9187b81a6ea579b41dce417f5caac953826d73 Mon Sep 17 00:00:00 2001
From: Chengpeng Yan <41809508+Reminiscent@users.noreply.github.com>
Date: Thu, 7 Sep 2023 02:00:15 +0800
Subject: [PATCH 18/61] Complete the missing metrics for
 files_created/bytes_written (#5120)

---
 pageserver/src/metrics.rs         |  6 ++++++
 pageserver/src/tenant/timeline.rs | 15 +++++----------
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs
index 1dc039056b..e024a9d77a 100644
--- a/pageserver/src/metrics.rs
+++ b/pageserver/src/metrics.rs
@@ -1165,6 +1165,12 @@ impl TimelineMetrics {
             ),
         }
     }
+
+    pub fn record_new_file_metrics(&self, sz: u64) {
+        self.resident_physical_size_gauge.add(sz);
+        self.num_persistent_files_created.inc_by(1);
+        self.persistent_bytes_written.inc_by(sz);
+    }
 }
 
 impl Drop for TimelineMetrics {
diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs
index 7b6c6dbfad..b616c3c572 100644
--- a/pageserver/src/tenant/timeline.rs
+++ b/pageserver/src/tenant/timeline.rs
@@ -2754,9 +2754,7 @@ impl Timeline {
 
                 // update metrics
                 let sz = l.layer_desc().file_size;
-                self.metrics.resident_physical_size_gauge.add(sz);
-                self.metrics.num_persistent_files_created.inc_by(1);
-                self.metrics.persistent_bytes_written.inc_by(sz);
+                self.metrics.record_new_file_metrics(sz);
             }
 
             guard.finish_flush_l0_layer(delta_layer_to_add, &frozen_layer);
@@ -3131,9 +3129,8 @@ impl Timeline {
                 LayerFileMetadata::new(metadata.len(), self.generation),
             );
 
-            self.metrics
-                .resident_physical_size_gauge
-                .add(metadata.len());
+            // update metrics
+            self.metrics.record_new_file_metrics(metadata.len());
             let l = Arc::new(l);
             l.access_stats().record_residence_event(
                 LayerResidenceStatus::Resident,
@@ -3815,10 +3812,8 @@ impl Timeline {
                 )?;
             }
 
-            // update the timeline's physical size
-            self.metrics
-                .resident_physical_size_gauge
-                .add(metadata.len());
+            // update metrics, including the timeline's physical size
+            self.metrics.record_new_file_metrics(metadata.len());
 
             new_layer_paths.insert(
                 new_delta_path,

From 31e1568deeedd17d4d0aaa1a03779eed757bb640 Mon Sep 17 00:00:00 2001
From: duguorong009 <80258679+duguorong009@users.noreply.github.com>
Date: Thu, 7 Sep 2023 02:31:49 +0800
Subject: [PATCH 19/61] refactor(pageserver): refactor pageserver router state
 creation (#5165)

Fixes #3894 by:
- Refactor the pageserver router creation flow
- Create the router state in `pageserver/src/bin/pageserver.rs`
---
 pageserver/src/bin/pageserver.rs | 23 +++++++++++++----------
 pageserver/src/http/routes.rs    | 21 ++++-----------------
 2 files changed, 17 insertions(+), 27 deletions(-)

diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs
index bf0ad760cb..6103f4c885 100644
--- a/pageserver/src/bin/pageserver.rs
+++ b/pageserver/src/bin/pageserver.rs
@@ -477,16 +477,19 @@ fn start_pageserver(
     {
         let _rt_guard = MGMT_REQUEST_RUNTIME.enter();
 
-        let router = http::make_router(
-            conf,
-            launch_ts,
-            http_auth,
-            broker_client.clone(),
-            remote_storage,
-            disk_usage_eviction_state,
-        )?
-        .build()
-        .map_err(|err| anyhow!(err))?;
+        let router_state = Arc::new(
+            http::routes::State::new(
+                conf,
+                http_auth.clone(),
+                remote_storage,
+                broker_client.clone(),
+                disk_usage_eviction_state,
+            )
+            .context("Failed to initialize router state")?,
+        );
+        let router = http::make_router(router_state, launch_ts, http_auth.clone())?
+            .build()
+            .map_err(|err| anyhow!(err))?;
         let service = utils::http::RouterService::new(router).unwrap();
         let server = hyper::Server::from_tcp(http_listener)?
             .serve(service)
diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs
index 286aa3dfb0..c661fe9e3f 100644
--- a/pageserver/src/http/routes.rs
+++ b/pageserver/src/http/routes.rs
@@ -12,7 +12,6 @@ use pageserver_api::models::{
     DownloadRemoteLayersTaskSpawnRequest, TenantAttachRequest, TenantLoadRequest,
 };
 use remote_storage::GenericRemoteStorage;
-use storage_broker::BrokerClientChannel;
 use tenant_size_model::{SizeResult, StorageModel};
 use tokio_util::sync::CancellationToken;
 use tracing::*;
@@ -55,7 +54,7 @@ use utils::{
 // Imports only used for testing APIs
 use super::models::ConfigureFailpointsRequest;
 
-struct State {
+pub struct State {
     conf: &'static PageServerConf,
     auth: Option<Arc<JwtAuth>>,
     allowlist_routes: Vec<Uri>,
@@ -65,7 +64,7 @@ struct State {
 }
 
 impl State {
-    fn new(
+    pub fn new(
         conf: &'static PageServerConf,
         auth: Option<Arc<JwtAuth>>,
         remote_storage: Option<GenericRemoteStorage>,
@@ -1354,12 +1353,9 @@ where
 }
 
 pub fn make_router(
-    conf: &'static PageServerConf,
+    state: Arc<State>,
     launch_ts: &'static LaunchTimestamp,
     auth: Option<Arc<JwtAuth>>,
-    broker_client: BrokerClientChannel,
-    remote_storage: Option<GenericRemoteStorage>,
-    disk_usage_eviction_state: Arc<disk_usage_eviction_task::State>,
 ) -> anyhow::Result<RouterBuilder<hyper::Body, ApiError>> {
     let spec = include_bytes!("openapi_spec.yml");
     let mut router = attach_openapi_ui(endpoint::make_router(), spec, "/swagger.yml", "/v1/doc");
@@ -1383,16 +1379,7 @@ pub fn make_router(
     );
 
     Ok(router
-        .data(Arc::new(
-            State::new(
-                conf,
-                auth,
-                remote_storage,
-                broker_client,
-                disk_usage_eviction_state,
-            )
-            .context("Failed to initialize router state")?,
-        ))
+        .data(state)
         .get("/v1/status", |r| api_handler(r, status_handler))
         .put("/v1/failpoints", |r| {
             testing_api_handler("manage failpoints", r, failpoints_handler)

From 3a966852aac317de30cd399dca38eb0c0c589635 Mon Sep 17 00:00:00 2001
From: Joonas Koivunen <joonas@neon.tech>
Date: Wed, 6 Sep 2023 21:40:00 +0300
Subject: [PATCH 20/61] doc: tests expect lsof (#5226)

On a clean system `lsof` needs to be installed. Add it to the list just
to keep things nice and copy-pasteable (except for poetry).
---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 5780608949..d948a92062 100644
--- a/README.md
+++ b/README.md
@@ -29,18 +29,18 @@ See developer documentation in [SUMMARY.md](/docs/SUMMARY.md) for more informati
 ```bash
 apt install build-essential libtool libreadline-dev zlib1g-dev flex bison libseccomp-dev \
 libssl-dev clang pkg-config libpq-dev cmake postgresql-client protobuf-compiler \
-libcurl4-openssl-dev openssl python-poetry
+libcurl4-openssl-dev openssl python-poetry lsof
 ```
 * On Fedora, these packages are needed:
 ```bash
 dnf install flex bison readline-devel zlib-devel openssl-devel \
   libseccomp-devel perl clang cmake postgresql postgresql-contrib protobuf-compiler \
-  protobuf-devel libcurl-devel openssl poetry
+  protobuf-devel libcurl-devel openssl poetry lsof
 ```
 * On Arch based systems, these packages are needed:
 ```bash
 pacman -S base-devel readline zlib libseccomp openssl clang \
-postgresql-libs cmake postgresql protobuf curl
+postgresql-libs cmake postgresql protobuf curl lsof
 ```
 
 Building Neon requires 3.15+ version of `protoc` (protobuf-compiler). If your distribution provides an older version, you can install a newer version from [here](https://github.com/protocolbuffers/protobuf/releases).

From 6243b44dea61efec7961ef7fd4e01447b250bd22 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arpad=20M=C3=BCller?= <arpad-m@users.noreply.github.com>
Date: Wed, 6 Sep 2023 20:54:57 +0200
Subject: [PATCH 21/61] Remove Virtual from FileBlockReaderVirtual variant name
 (#5225)

With #5181, the generics for `FileBlockReader` have been removed, so
having a `Virtual` postfix makes less sense now.
---
 pageserver/src/tenant/block_io.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pageserver/src/tenant/block_io.rs b/pageserver/src/tenant/block_io.rs
index abb636579d..ab5b5e9fca 100644
--- a/pageserver/src/tenant/block_io.rs
+++ b/pageserver/src/tenant/block_io.rs
@@ -71,7 +71,7 @@ impl<'a> Deref for BlockLease<'a> {
 ///
 /// Unlike traits, we also support the read function to be async though.
 pub(crate) enum BlockReaderRef<'a> {
-    FileBlockReaderVirtual(&'a FileBlockReader),
+    FileBlockReader(&'a FileBlockReader),
     EphemeralFile(&'a EphemeralFile),
     Adapter(Adapter<&'a DeltaLayerInner>),
     #[cfg(test)]
@@ -85,7 +85,7 @@ impl<'a> BlockReaderRef<'a> {
     async fn read_blk(&self, blknum: u32) -> Result<BlockLease, std::io::Error> {
         use BlockReaderRef::*;
         match self {
-            FileBlockReaderVirtual(r) => r.read_blk(blknum).await,
+            FileBlockReader(r) => r.read_blk(blknum).await,
             EphemeralFile(r) => r.read_blk(blknum).await,
             Adapter(r) => r.read_blk(blknum).await,
             #[cfg(test)]
@@ -124,7 +124,7 @@ impl<'a> BlockCursor<'a> {
     // Needed by cli
     pub fn new_fileblockreader(reader: &'a FileBlockReader) -> Self {
         BlockCursor {
-            reader: BlockReaderRef::FileBlockReaderVirtual(reader),
+            reader: BlockReaderRef::FileBlockReader(reader),
         }
     }
 
@@ -197,7 +197,7 @@ impl FileBlockReader {
 
 impl BlockReader for FileBlockReader {
     fn block_cursor(&self) -> BlockCursor<'_> {
-        BlockCursor::new(BlockReaderRef::FileBlockReaderVirtual(self))
+        BlockCursor::new(BlockReaderRef::FileBlockReader(self))
     }
 }
 

From 7ba0f5c08d85aa8fd6ca9981f1857e3b832ab145 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arpad=20M=C3=BCller?= <arpad-m@users.noreply.github.com>
Date: Wed, 6 Sep 2023 21:44:54 +0200
Subject: [PATCH 22/61] Improve comment in page cache (#5220)

It was easy to interpret comment in the page cache initialization code
to be about justifying why we leak here at all, not just why this
specific type of leaking is done (which the comment was actually meant
to describe).

See
https://github.com/neondatabase/neon/pull/5125#discussion_r1308445993

---------

Co-authored-by: Joonas Koivunen <joonas@neon.tech>
---
 pageserver/src/page_cache.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/pageserver/src/page_cache.rs b/pageserver/src/page_cache.rs
index fb1c5fc485..9cc1bf35dc 100644
--- a/pageserver/src/page_cache.rs
+++ b/pageserver/src/page_cache.rs
@@ -799,8 +799,9 @@ impl PageCache {
     fn new(num_pages: usize) -> Self {
         assert!(num_pages > 0, "page cache size must be > 0");
 
-        // We use Box::leak here and into_boxed_slice to avoid leaking uninitialized
-        // memory that Vec's might contain.
+        // We could use Vec::leak here, but that potentially also leaks
+        // uninitialized reserved capacity. With into_boxed_slice and Box::leak
+        // this is avoided.
         let page_buffer = Box::leak(vec![0u8; num_pages * PAGE_SZ].into_boxed_slice());
 
         let size_metrics = &crate::metrics::PAGE_CACHE_SIZE;

From 706977fb7714aa87516ac478db183d0304aa2287 Mon Sep 17 00:00:00 2001
From: duguorong009 <80258679+duguorong009@users.noreply.github.com>
Date: Thu, 7 Sep 2023 19:17:18 +0800
Subject: [PATCH 23/61] fix(pageserver): add the walreceiver state to tenant
 timeline GET api endpoint (#5196)

Add a `walreceiver_state` field to `TimelineInfo` (response of `GET /v1/tenant/:tenant_id/timeline/:timeline_id`) and while doing that, refactor out a common `Timeline::walreceiver_state(..)`. No OpenAPI changes, because this is an internal debugging addition.

Fixes #3115.

Co-authored-by: Joonas Koivunen <joonas.koivunen@gmail.com>
---
 libs/pageserver_api/src/models.rs             |  2 ++
 pageserver/src/http/routes.rs                 |  4 ++++
 pageserver/src/tenant/timeline.rs             | 20 ++++++++++---------
 pageserver/src/tenant/timeline/walreceiver.rs |  2 +-
 4 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs
index 340463afa7..f354296be2 100644
--- a/libs/pageserver_api/src/models.rs
+++ b/libs/pageserver_api/src/models.rs
@@ -381,6 +381,8 @@ pub struct TimelineInfo {
     pub pg_version: u32,
 
     pub state: TimelineState,
+
+    pub walreceiver_status: String,
 }
 
 #[derive(Debug, Clone, Serialize)]
diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs
index c661fe9e3f..a8e914ba08 100644
--- a/pageserver/src/http/routes.rs
+++ b/pageserver/src/http/routes.rs
@@ -285,6 +285,8 @@ async fn build_timeline_info_common(
     let state = timeline.current_state();
     let remote_consistent_lsn = timeline.get_remote_consistent_lsn().unwrap_or(Lsn(0));
 
+    let walreceiver_status = timeline.walreceiver_status();
+
     let info = TimelineInfo {
         tenant_id: timeline.tenant_id,
         timeline_id: timeline.timeline_id,
@@ -305,6 +307,8 @@ async fn build_timeline_info_common(
         pg_version: timeline.pg_version,
 
         state,
+
+        walreceiver_status,
     };
     Ok(info)
 }
diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs
index b616c3c572..cb8b842cf6 100644
--- a/pageserver/src/tenant/timeline.rs
+++ b/pageserver/src/tenant/timeline.rs
@@ -585,15 +585,7 @@ impl Timeline {
             Err(e) => {
                 // don't count the time spent waiting for lock below, and also in walreceiver.status(), towards the wait_lsn_time_histo
                 drop(_timer);
-                let walreceiver_status = {
-                    match &*self.walreceiver.lock().unwrap() {
-                        None => "stopping or stopped".to_string(),
-                        Some(walreceiver) => match walreceiver.status() {
-                            Some(status) => status.to_human_readable_string(),
-                            None => "Not active".to_string(),
-                        },
-                    }
-                };
+                let walreceiver_status = self.walreceiver_status();
                 Err(anyhow::Error::new(e).context({
                     format!(
                         "Timed out while waiting for WAL record at LSN {} to arrive, last_record_lsn {} disk consistent LSN={}, WalReceiver status: {}",
@@ -607,6 +599,16 @@ impl Timeline {
         }
     }
 
+    pub(crate) fn walreceiver_status(&self) -> String {
+        match &*self.walreceiver.lock().unwrap() {
+            None => "stopping or stopped".to_string(),
+            Some(walreceiver) => match walreceiver.status() {
+                Some(status) => status.to_human_readable_string(),
+                None => "Not active".to_string(),
+            },
+        }
+    }
+
     /// Check that it is valid to request operations with that lsn.
     pub fn check_lsn_is_in_scope(
         &self,
diff --git a/pageserver/src/tenant/timeline/walreceiver.rs b/pageserver/src/tenant/timeline/walreceiver.rs
index ccff735c3c..842bc3675c 100644
--- a/pageserver/src/tenant/timeline/walreceiver.rs
+++ b/pageserver/src/tenant/timeline/walreceiver.rs
@@ -135,7 +135,7 @@ impl WalReceiver {
         .await;
     }
 
-    pub(super) fn status(&self) -> Option<ConnectionManagerStatus> {
+    pub(crate) fn status(&self) -> Option<ConnectionManagerStatus> {
         self.manager_status.read().unwrap().clone()
     }
 }

From f8a91e792c2b4d4e58f142b3f93eefcf4a81302a Mon Sep 17 00:00:00 2001
From: Alexander Bayandin <alexander@neon.tech>
Date: Thu, 7 Sep 2023 14:21:01 +0100
Subject: [PATCH 24/61] Even better handling of `approved-for-ci-run` label
 (#5227)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Problem

We've got `approved-for-ci-run` to work 🎉
But it's still a bit rough, this PR should improve the UX for external
contributors.

## Summary of changes
- `build_and_test.yml`: add `check-permissions` job, which fails if PR is
created from a fork. Make all jobs in the workflow to be dependant on
`check-permission` to fail fast
- `approved-for-ci-run.yml`: add `cleanup` job to close `ci-run/pr-*`
PRs and delete linked branches when the parent PR is closed
- `approved-for-ci-run.yml`: fix the layout for the `ci-run/pr-*` PR
description
- GitHub Autocomment: add a comment with tests result to the original PR
(instead of a PR from `ci-run/pr-*` )
---
 .github/workflows/approved-for-ci-run.yml | 38 +++++++++++---
 .github/workflows/build_and_test.yml      | 64 +++++++++++++++++------
 scripts/comment-test-report.js            | 14 ++++-
 3 files changed, 90 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/approved-for-ci-run.yml b/.github/workflows/approved-for-ci-run.yml
index 81955a602e..9e9a46b354 100644
--- a/.github/workflows/approved-for-ci-run.yml
+++ b/.github/workflows/approved-for-ci-run.yml
@@ -66,13 +66,37 @@ jobs:
           GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
         run: |
           HEAD="ci-run/pr-${PR_NUMBER}"
-          BODY="This Pull Request was create automatically to run CI pipeline for #${PR_NUMBER}.\n\nPlease do not alter or merge/close it.\n\nFeel free to comment the original PR."
+          cat << EOF > body.md
+            This Pull Request is created automatically to run the CI pipeline for #${PR_NUMBER}
 
-          ALREADY_CREATED=$(gh pr --repo "${GITHUB_REPOSITORY}" list --head "${HEAD}" --base "main" --json "number" --jq '.[].number')
+            Please do not alter or merge/close it.
+
+            Feel free to review/comment/discuss the original PR #${PR_NUMBER}.
+          EOF
+
+          ALREADY_CREATED="$(gh pr --repo '${GITHUB_REPOSITORY}' list --head '${HEAD}' --base 'main' --json 'number' --jq '.[].number')"
           if [ -z "${ALREADY_CREATED}" ]; then
-            gh pr --repo "${GITHUB_REPOSITORY}" create  --title "CI run for PR #${PR_NUMBER}" \
-                                                        --body "${BODY}" \
-                                                        --head "${HEAD}" \
-                                                        --base "main" \
-                                                        --draft
+            gh pr --repo "${GITHUB_REPOSITORY}" create --title "CI run for PR #${PR_NUMBER}" \
+                                                       --body-file "body.md" \
+                                                       --head "${HEAD}" \
+                                                       --base "main" \
+                                                       --draft
+          fi
+
+  cleanup:
+    # Close PRs and delete branchs if the original PR is closed.
+
+    if: |
+      github.event.action == 'closed' &&
+      github.event.pull_request.head.repo.full_name != github.repository
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - run: |
+          HEAD="ci-run/pr-${PR_NUMBER}"
+
+          CLOSED="$(gh pr --repo '${GITHUB_REPOSITORY}' list --head '${HEAD}' --json 'closed' --jq '.[].closed')"
+          if [ "${CLOSED}" != "false" ]; then
+            gh pr --repo "${GITHUB_REPOSITORY}" close "ci-run/pr-${{ github.event.pull_request.number }}" --delete-branch
           fi
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index d03f8c4365..88fd5a7d7c 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -23,7 +23,30 @@ env:
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }}
 
 jobs:
+  check-permissions:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Disallow PRs from forks
+      if: |
+        github.event_name == 'pull_request' &&
+        github.event.pull_request.head.repo.full_name != github.repository
+
+      run: |
+        if [ "${{ contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) }}" = "true" ]; then
+          MESSAGE="Please create a PR from a branch of ${GITHUB_REPOSITORY} instead of a fork"
+        else
+          MESSAGE="The PR should be reviewed and labelled with `approved-for-ci-run` to trigger a CI run"
+        fi
+
+        echo >&2 "We don't run CI for PRs from forks"
+        echo >&2 "${MESSAGE}"
+
+        exit 1
+
+
   tag:
+    needs: [ check-permissions ]
     runs-on: [ self-hosted, gen3, small ]
     container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned
     outputs:
@@ -52,6 +75,7 @@ jobs:
         id: build-tag
 
   check-codestyle-python:
+    needs: [ check-permissions ]
     runs-on: [ self-hosted, gen3, small ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
@@ -84,6 +108,7 @@ jobs:
         run: poetry run mypy .
 
   check-codestyle-rust:
+    needs: [ check-permissions ]
     runs-on: [ self-hosted, gen3, large ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
@@ -150,6 +175,7 @@ jobs:
         run: cargo deny check
 
   build-neon:
+    needs: [ check-permissions ]
     runs-on: [ self-hosted, gen3, large ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
@@ -347,12 +373,12 @@ jobs:
         uses: ./.github/actions/save-coverage-data
 
   regress-tests:
+    needs: [ check-permissions, build-neon ]
     runs-on: [ self-hosted, gen3, large ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       # Default shared memory is 64mb
       options: --init --shm-size=512mb
-    needs: [ build-neon ]
     strategy:
       fail-fast: false
       matrix:
@@ -385,12 +411,12 @@ jobs:
         uses: ./.github/actions/save-coverage-data
 
   benchmarks:
+    needs: [ check-permissions, build-neon ]
     runs-on: [ self-hosted, gen3, small ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       # Default shared memory is 64mb
       options: --init --shm-size=512mb
-    needs: [ build-neon ]
     if: github.ref_name == 'main' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks')
     strategy:
       fail-fast: false
@@ -417,12 +443,13 @@ jobs:
       # while coverage is currently collected for the debug ones
 
   create-test-report:
+    needs: [ check-permissions, regress-tests, coverage-report, benchmarks ]
+    if: ${{ !cancelled() }}
+
     runs-on: [ self-hosted, gen3, small ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init
-    needs: [ regress-tests, coverage-report, benchmarks ]
-    if: ${{ !cancelled() }}
 
     steps:
       - uses: actions/checkout@v3
@@ -463,11 +490,12 @@ jobs:
             })
 
   coverage-report:
+    needs: [ check-permissions, regress-tests ]
+
     runs-on: [ self-hosted, gen3, small ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
       options: --init
-    needs: [ regress-tests ]
     strategy:
       fail-fast: false
       matrix:
@@ -582,11 +610,11 @@ jobs:
             })
 
   trigger-e2e-tests:
+    needs: [ check-permissions, promote-images, tag ]
     runs-on: [ self-hosted, gen3, small ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned
       options: --init
-    needs: [ promote-images, tag ]
     steps:
       - name: Set PR's status to pending and request a remote CI test
         run: |
@@ -627,8 +655,8 @@ jobs:
             }"
 
   neon-image:
+    needs: [ check-permissions, tag ]
     runs-on: [ self-hosted, gen3, large ]
-    needs: [ tag ]
     container: gcr.io/kaniko-project/executor:v1.9.2-debug
     defaults:
       run:
@@ -675,7 +703,7 @@ jobs:
 
   compute-tools-image:
     runs-on: [ self-hosted, gen3, large ]
-    needs: [ tag ]
+    needs: [ check-permissions, tag ]
     container: gcr.io/kaniko-project/executor:v1.9.2-debug
     defaults:
       run:
@@ -720,13 +748,13 @@ jobs:
         run: rm -rf ~/.ecr
 
   compute-node-image:
+    needs: [ check-permissions, tag ]
     runs-on: [ self-hosted, gen3, large ]
     container:
       image: gcr.io/kaniko-project/executor:v1.9.2-debug
       # Workaround for "Resolving download.osgeo.org (download.osgeo.org)... failed: Temporary failure in name resolution.""
       # Should be prevented by https://github.com/neondatabase/neon/issues/4281
       options: --add-host=download.osgeo.org:140.211.15.30
-    needs: [ tag ]
     strategy:
       fail-fast: false
       matrix:
@@ -779,8 +807,8 @@ jobs:
         run: rm -rf ~/.ecr
 
   vm-compute-node-image:
+    needs: [ check-permissions, tag, compute-node-image ]
     runs-on: [ self-hosted, gen3, large ]
-    needs: [ tag, compute-node-image ]
     strategy:
       fail-fast: false
       matrix:
@@ -821,7 +849,7 @@ jobs:
           docker push 369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}}
 
   test-images:
-    needs: [ tag, neon-image, compute-node-image, compute-tools-image ]
+    needs: [ check-permissions, tag, neon-image, compute-node-image, compute-tools-image ]
     runs-on: [ self-hosted, gen3, small ]
 
     steps:
@@ -864,8 +892,8 @@ jobs:
           docker compose -f ./docker-compose/docker-compose.yml down
 
   promote-images:
+    needs: [ check-permissions, tag, test-images, vm-compute-node-image ]
     runs-on: [ self-hosted, gen3, small ]
-    needs: [ tag, test-images, vm-compute-node-image ]
     container: golang:1.19-bullseye
     # Don't add if-condition here.
     # The job should always be run because we have dependant other jobs that shouldn't be skipped
@@ -937,8 +965,8 @@ jobs:
         run: rm -rf ~/.ecr
 
   trigger-custom-extensions-build-and-wait:
+    needs: [ check-permissions, tag ]
     runs-on: ubuntu-latest
-    needs: [ tag ]
     steps:
       - name: Set PR's status to pending and request a remote CI test
         run: |
@@ -1011,10 +1039,11 @@ jobs:
           exit 1
 
   deploy:
+    needs: [ check-permissions, promote-images, tag, regress-tests, trigger-custom-extensions-build-and-wait ]
+    if: ( github.ref_name == 'main' || github.ref_name == 'release' ) && github.event_name != 'workflow_dispatch'
+
     runs-on: [ self-hosted, gen3, small ]
     container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest
-    needs: [ promote-images, tag, regress-tests, trigger-custom-extensions-build-and-wait ]
-    if: ( github.ref_name == 'main' || github.ref_name == 'release' ) && github.event_name != 'workflow_dispatch'
     steps:
       - name: Fix git ownership
         run: |
@@ -1060,12 +1089,13 @@ jobs:
             })
 
   promote-compatibility-data:
+    needs: [ check-permissions, promote-images, tag, regress-tests ]
+    if: github.ref_name == 'release'
+
     runs-on: [ self-hosted, gen3, small ]
     container:
       image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned
       options: --init
-    needs: [ promote-images, tag, regress-tests ]
-    if: github.ref_name == 'release' && github.event_name != 'workflow_dispatch'
     steps:
       - name: Promote compatibility snapshot for the release
         env:
diff --git a/scripts/comment-test-report.js b/scripts/comment-test-report.js
index 055e3c1b2e..92a016e053 100755
--- a/scripts/comment-test-report.js
+++ b/scripts/comment-test-report.js
@@ -205,8 +205,18 @@ const parseCoverageSummary = async ({ summaryJsonUrl, coverageUrl, fetch }) => {
 }
 
 module.exports = async ({ github, context, fetch, report, coverage }) => {
+    // Which PR to comment (for ci-run/pr-* it will comment the parent PR, not the ci-run/pr-* PR)
+    let prToComment
+    const branchName = context.payload.pull_request.base.ref.replace(/^refs\/heads\//, "")
+    const match = branchName.match(/^ci-run\/pr-(?<prNumber>\d+)$/)?.groups
+    if (match) {
+        ({ prNumber } = match)
+        prToComment = parseInt(prNumber, 10)
+    } else {
+        prToComment = context.payload.number
+    }
     // Marker to find the comment in the subsequent runs
-    const startMarker = `<!--AUTOMATIC COMMENT START #${context.payload.number}-->`
+    const startMarker = `<!--AUTOMATIC COMMENT START #${prToComment}-->`
     // If we run the script in the PR or in the branch (main/release/...)
     const isPullRequest = !!context.payload.pull_request
     // Latest commit in PR or in the branch
@@ -267,7 +277,7 @@ module.exports = async ({ github, context, fetch, report, coverage }) => {
         listCommentsFn   = github.rest.issues.listComments
         updateCommentFn  = github.rest.issues.updateComment
         issueNumberOrSha = {
-            issue_number: context.payload.number,
+            issue_number: prToComment,
         }
     } else {
         updateCommentFn  = github.rest.repos.updateCommitComment

From 1fee69371bc2384e960a7399062a159b3d0deda0 Mon Sep 17 00:00:00 2001
From: Alexander Bayandin <alexander@neon.tech>
Date: Thu, 7 Sep 2023 14:21:38 +0100
Subject: [PATCH 25/61] Update `plv8` to 3.1.8 (#5230)

## Problem

We likely need this to support Postgres 16
It's also been asked by a user
https://github.com/neondatabase/neon/discussions/5042

The latest version is 3.2.0, but it requires some changes in the build
script (which I haven't checked, but it didn't work right away)

## Summary of changes
```
3.1.8       2023-08-01
            - force v8 to compile in release mode

3.1.7       2023-06-26
            - fix byteoffset issue with arraybuffers
            - support postgres 16 beta

3.1.6       2023-04-08
            - fix crash issue on fetch apply
            - fix interrupt issue
```
From https://github.com/plv8/plv8/blob/v3.1.8/Changes
---
 Dockerfile.compute-node | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node
index 0416a98450..c71fd8619b 100644
--- a/Dockerfile.compute-node
+++ b/Dockerfile.compute-node
@@ -124,8 +124,8 @@ COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
 RUN apt update && \
     apt install -y ninja-build python3-dev libncurses5 binutils clang
 
-RUN wget https://github.com/plv8/plv8/archive/refs/tags/v3.1.5.tar.gz -O plv8.tar.gz && \
-    echo "1e108d5df639e4c189e1c5bdfa2432a521c126ca89e7e5a969d46899ca7bf106 plv8.tar.gz" | sha256sum --check && \
+RUN wget https://github.com/plv8/plv8/archive/refs/tags/v3.1.8.tar.gz -O plv8.tar.gz && \
+    echo "92b10c7db39afdae97ff748c9ec54713826af222c459084ad002571b79eb3f49 plv8.tar.gz" | sha256sum --check && \
     mkdir plv8-src && cd plv8-src && tar xvzf ../plv8.tar.gz --strip-components=1 -C . && \
     export PATH="/usr/local/pgsql/bin:$PATH" && \
     make DOCKER=1 -j $(getconf _NPROCESSORS_ONLN) install && \

From 485a2cfdd3f0cd4f3ed17129e9ebcccaea30b0bc Mon Sep 17 00:00:00 2001
From: Rahul Modpur <rmodpur2@gmail.com>
Date: Thu, 7 Sep 2023 19:04:22 +0530
Subject: [PATCH 26/61] Fix pg_config version parsing (#5200)

## Problem
Fix pg_config version parsing

## Summary of changes
Use regex to capture major version of postgres
#5146
---
 compute_tools/src/extension_server.rs | 53 +++++++++++++++++++++++++--
 1 file changed, 49 insertions(+), 4 deletions(-)

diff --git a/compute_tools/src/extension_server.rs b/compute_tools/src/extension_server.rs
index 54c22026e7..bf32f4a517 100644
--- a/compute_tools/src/extension_server.rs
+++ b/compute_tools/src/extension_server.rs
@@ -74,6 +74,7 @@ More specifically, here is an example ext_index.json
 use anyhow::Context;
 use anyhow::{self, Result};
 use compute_api::spec::RemoteExtSpec;
+use regex::Regex;
 use remote_storage::*;
 use serde_json;
 use std::io::Read;
@@ -108,14 +109,58 @@ pub fn get_pg_version(pgbin: &str) -> String {
     // pg_config --version returns a (platform specific) human readable string
     // such as "PostgreSQL 15.4". We parse this to v14/v15
     let human_version = get_pg_config("--version", pgbin);
-    if human_version.contains("15") {
-        return "v15".to_string();
-    } else if human_version.contains("14") {
-        return "v14".to_string();
+    return parse_pg_version(&human_version).to_string();
+}
+
+fn parse_pg_version(human_version: &str) -> &str {
+    match Regex::new(r"(?<major>\d+)\.(?<minor>\d+)")
+        .unwrap()
+        .captures(human_version)
+    {
+        Some(captures) if captures.len() == 3 => match &captures["major"] {
+            "14" => return "v14",
+            "15" => return "v15",
+            _ => {}
+        },
+        _ => {}
     }
     panic!("Unsuported postgres version {human_version}");
 }
 
+#[cfg(test)]
+mod tests {
+    use super::parse_pg_version;
+
+    #[test]
+    fn test_parse_pg_version() {
+        assert_eq!(parse_pg_version("PostgreSQL 15.4"), "v15");
+        assert_eq!(parse_pg_version("PostgreSQL 15.14"), "v15");
+        assert_eq!(
+            parse_pg_version("PostgreSQL 15.4 (Ubuntu 15.4-0ubuntu0.23.04.1)"),
+            "v15"
+        );
+
+        assert_eq!(parse_pg_version("PostgreSQL 14.15"), "v14");
+        assert_eq!(parse_pg_version("PostgreSQL 14.0"), "v14");
+        assert_eq!(
+            parse_pg_version("PostgreSQL 14.9 (Debian 14.9-1.pgdg120+1"),
+            "v14"
+        );
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_parse_pg_unsupported_version() {
+        parse_pg_version("PostgreSQL 13.14");
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_parse_pg_incorrect_version_format() {
+        parse_pg_version("PostgreSQL 14");
+    }
+}
+
 // download the archive for a given extension,
 // unzip it, and place files in the appropriate locations (share/lib)
 pub async fn download_extension(

From d15563f93bb36c8557cacf028bafc7fd2bb5e988 Mon Sep 17 00:00:00 2001
From: Alexander Bayandin <alexander@neon.tech>
Date: Thu, 7 Sep 2023 17:39:42 +0100
Subject: [PATCH 27/61] Misc workflows: fix quotes in bash (#5235)

---
 .github/workflows/approved-for-ci-run.yml |  2 +-
 .github/workflows/build_and_test.yml      |  2 +-
 scripts/comment-test-report.js            | 20 +++++++++++---------
 3 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/approved-for-ci-run.yml b/.github/workflows/approved-for-ci-run.yml
index 9e9a46b354..bcd042e608 100644
--- a/.github/workflows/approved-for-ci-run.yml
+++ b/.github/workflows/approved-for-ci-run.yml
@@ -74,7 +74,7 @@ jobs:
             Feel free to review/comment/discuss the original PR #${PR_NUMBER}.
           EOF
 
-          ALREADY_CREATED="$(gh pr --repo '${GITHUB_REPOSITORY}' list --head '${HEAD}' --base 'main' --json 'number' --jq '.[].number')"
+          ALREADY_CREATED="$(gh pr --repo ${GITHUB_REPOSITORY} list --head ${HEAD} --base main --json number --jq '.[].number')"
           if [ -z "${ALREADY_CREATED}" ]; then
             gh pr --repo "${GITHUB_REPOSITORY}" create --title "CI run for PR #${PR_NUMBER}" \
                                                        --body-file "body.md" \
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 88fd5a7d7c..bd57d5bd5a 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -36,7 +36,7 @@ jobs:
         if [ "${{ contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association) }}" = "true" ]; then
           MESSAGE="Please create a PR from a branch of ${GITHUB_REPOSITORY} instead of a fork"
         else
-          MESSAGE="The PR should be reviewed and labelled with `approved-for-ci-run` to trigger a CI run"
+          MESSAGE="The PR should be reviewed and labelled with 'approved-for-ci-run' to trigger a CI run"
         fi
 
         echo >&2 "We don't run CI for PRs from forks"
diff --git a/scripts/comment-test-report.js b/scripts/comment-test-report.js
index 92a016e053..ce92534dea 100755
--- a/scripts/comment-test-report.js
+++ b/scripts/comment-test-report.js
@@ -205,20 +205,22 @@ const parseCoverageSummary = async ({ summaryJsonUrl, coverageUrl, fetch }) => {
 }
 
 module.exports = async ({ github, context, fetch, report, coverage }) => {
+    // If we run the script in the PR or in the branch (main/release/...)
+    const isPullRequest = !!context.payload.pull_request
     // Which PR to comment (for ci-run/pr-* it will comment the parent PR, not the ci-run/pr-* PR)
     let prToComment
-    const branchName = context.payload.pull_request.base.ref.replace(/^refs\/heads\//, "")
-    const match = branchName.match(/^ci-run\/pr-(?<prNumber>\d+)$/)?.groups
-    if (match) {
-        ({ prNumber } = match)
-        prToComment = parseInt(prNumber, 10)
-    } else {
-        prToComment = context.payload.number
+    if (isPullRequest) {
+        const branchName = context.payload.pull_request.base.ref.replace(/^refs\/heads\//, "")
+        const match = branchName.match(/ci-run\/pr-(?<prNumber>\d+)/)?.groups
+        if (match) {
+            ({ prNumber } = match)
+            prToComment = parseInt(prNumber, 10)
+        } else {
+            prToComment = context.payload.number
+        }
     }
     // Marker to find the comment in the subsequent runs
     const startMarker = `<!--AUTOMATIC COMMENT START #${prToComment}-->`
-    // If we run the script in the PR or in the branch (main/release/...)
-    const isPullRequest = !!context.payload.pull_request
     // Latest commit in PR or in the branch
     const commitSha = isPullRequest ? context.payload.pull_request.head.sha : context.sha
     // Let users know that the comment is updated automatically

From c202f0ba1089403d8836b532247d43269d20bc2a Mon Sep 17 00:00:00 2001
From: Heikki Linnakangas <heikki@neon.tech>
Date: Thu, 7 Sep 2023 21:38:55 +0300
Subject: [PATCH 28/61] Update PostGIS to version 3.3.3 (#5236)

It's a good idea to keep up-to-date in general. One noteworthy change is
that PostGIS 3.3.3 adds support for PostgreSQL v16. We'll need that.

PostGIS 3.4.0 has already been released, and we should consider
upgrading to that. However, it's a major upgrade and requires running
"SELECT postgis_extensions_upgrade();" in each database, to upgrade the
catalogs. I don't want to deal with that right now.
---
 Dockerfile.compute-node | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node
index c71fd8619b..7b4632d268 100644
--- a/Dockerfile.compute-node
+++ b/Dockerfile.compute-node
@@ -74,8 +74,8 @@ RUN wget https://gitlab.com/Oslandia/SFCGAL/-/archive/v1.3.10/SFCGAL-v1.3.10.tar
 
 ENV PATH "/usr/local/pgsql/bin:$PATH"
 
-RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.2.tar.gz -O postgis.tar.gz && \
-    echo "9a2a219da005a1730a39d1959a1c7cec619b1efb009b65be80ffc25bad299068 postgis.tar.gz" | sha256sum --check && \
+RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.3.tar.gz -O postgis.tar.gz && \
+    echo "74eb356e3f85f14233791013360881b6748f78081cc688ff9d6f0f673a762d13 postgis.tar.gz" | sha256sum --check && \
     mkdir postgis-src && cd postgis-src && tar xvzf ../postgis.tar.gz --strip-components=1 -C . && \
     find /usr/local/pgsql -type f | sed 's|^/usr/local/pgsql/||' > /before.txt &&\
     ./autogen.sh && \

From e5adc4efb97b7adf5189b999c8f346519915f3e7 Mon Sep 17 00:00:00 2001
From: Heikki Linnakangas <heikki@neon.tech>
Date: Thu, 7 Sep 2023 21:39:12 +0300
Subject: [PATCH 29/61] Upgrade h3-pg to version 4.1.3. (#5237)

This includes v16 support.
---
 Dockerfile.compute-node | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node
index 7b4632d268..563a05c845 100644
--- a/Dockerfile.compute-node
+++ b/Dockerfile.compute-node
@@ -172,8 +172,8 @@ RUN wget https://github.com/uber/h3/archive/refs/tags/v4.1.0.tar.gz -O h3.tar.gz
     cp -R /h3/usr / && \
     rm -rf build
 
-RUN wget https://github.com/zachasme/h3-pg/archive/refs/tags/v4.1.2.tar.gz -O h3-pg.tar.gz && \
-    echo "c135aa45999b2ad1326d2537c1cadef96d52660838e4ca371706c08fdea1a956 h3-pg.tar.gz" | sha256sum --check && \
+RUN wget https://github.com/zachasme/h3-pg/archive/refs/tags/v4.1.3.tar.gz -O h3-pg.tar.gz && \
+    echo "5c17f09a820859ffe949f847bebf1be98511fb8f1bd86f94932512c00479e324 h3-pg.tar.gz" | sha256sum --check && \
     mkdir h3-pg-src && cd h3-pg-src && tar xvzf ../h3-pg.tar.gz --strip-components=1 -C . && \
     export PATH="/usr/local/pgsql/bin:$PATH" && \
     make -j $(getconf _NPROCESSORS_ONLN) && \

From d206655a63996910f887d0acbfa8fd7e957ec9f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arpad=20M=C3=BCller?= <arpad-m@users.noreply.github.com>
Date: Fri, 8 Sep 2023 00:50:50 +0200
Subject: [PATCH 30/61] Make VirtualFile::{open, open_with_options,
 create,sync_all,with_file} async fn (#5224)

## Problem

Once we use async file system APIs for `VirtualFile`, these functions
will also need to be async fn.

## Summary of changes

Makes the functions `open, open_with_options, create,sync_all,with_file`
of `VirtualFile` async fn, including all functions that call it. Like in
the prior PRs, the actual I/O operations are not using async APIs yet,
as per request in the #4743 epic.

We switch towards not using `VirtualFile` in the par_fsync module,
hopefully this is only temporary until we can actually do fully async
I/O in `VirtualFile`. This might cause us to exhaust fd limits in the
tests, but it should only be an issue for the local developer as we have
high ulimits in prod.

This PR is a follow-up of #5189, #5190, #5195, and #5203. Part of #4743.
---
 pageserver/ctl/src/layer_map_analyzer.rs      |  2 +-
 pageserver/ctl/src/layers.rs                  |  2 +-
 pageserver/src/tenant/blob_io.rs              |  4 +-
 pageserver/src/tenant/ephemeral_file.rs       |  7 +-
 pageserver/src/tenant/par_fsync.rs            |  5 +-
 .../src/tenant/storage_layer/delta_layer.rs   |  5 +-
 .../src/tenant/storage_layer/image_layer.rs   |  6 +-
 .../tenant/storage_layer/inmemory_layer.rs    |  4 +-
 pageserver/src/tenant/timeline.rs             | 16 ++--
 .../src/tenant/timeline/layer_manager.rs      |  4 +-
 pageserver/src/virtual_file.rs                | 84 ++++++++++++-------
 11 files changed, 86 insertions(+), 53 deletions(-)

diff --git a/pageserver/ctl/src/layer_map_analyzer.rs b/pageserver/ctl/src/layer_map_analyzer.rs
index 32d0d1bed2..495dae87e3 100644
--- a/pageserver/ctl/src/layer_map_analyzer.rs
+++ b/pageserver/ctl/src/layer_map_analyzer.rs
@@ -97,7 +97,7 @@ pub(crate) fn parse_filename(name: &str) -> Option<LayerFile> {
 
 // Finds the max_holes largest holes, ignoring any that are smaller than MIN_HOLE_LENGTH"
 async fn get_holes(path: &Path, max_holes: usize) -> Result<Vec<Hole>> {
-    let file = FileBlockReader::new(VirtualFile::open(path)?);
+    let file = FileBlockReader::new(VirtualFile::open(path).await?);
     let summary_blk = file.read_blk(0).await?;
     let actual_summary = Summary::des_prefix(summary_blk.as_ref())?;
     let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new(
diff --git a/pageserver/ctl/src/layers.rs b/pageserver/ctl/src/layers.rs
index ff2044653a..33a6f197cf 100644
--- a/pageserver/ctl/src/layers.rs
+++ b/pageserver/ctl/src/layers.rs
@@ -48,7 +48,7 @@ async fn read_delta_file(path: impl AsRef<Path>) -> Result<()> {
     let path = path.as_ref();
     virtual_file::init(10);
     page_cache::init(100);
-    let file = FileBlockReader::new(VirtualFile::open(path)?);
+    let file = FileBlockReader::new(VirtualFile::open(path).await?);
     let summary_blk = file.read_blk(0).await?;
     let actual_summary = Summary::des_prefix(summary_blk.as_ref())?;
     let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new(
diff --git a/pageserver/src/tenant/blob_io.rs b/pageserver/src/tenant/blob_io.rs
index 71db8d2978..4fad1f3c14 100644
--- a/pageserver/src/tenant/blob_io.rs
+++ b/pageserver/src/tenant/blob_io.rs
@@ -238,7 +238,7 @@ mod tests {
         // Write part (in block to drop the file)
         let mut offsets = Vec::new();
         {
-            let file = VirtualFile::create(&path)?;
+            let file = VirtualFile::create(&path).await?;
             let mut wtr = BlobWriter::<BUFFERED>::new(file, 0);
             for blob in blobs.iter() {
                 let offs = wtr.write_blob(blob).await?;
@@ -251,7 +251,7 @@ mod tests {
             wtr.flush_buffer().await?;
         }
 
-        let file = VirtualFile::open(&path)?;
+        let file = VirtualFile::open(&path).await?;
         let rdr = BlockReaderRef::VirtualFile(&file);
         let rdr = BlockCursor::new(rdr);
         for (idx, (blob, offset)) in blobs.iter().zip(offsets.iter()).enumerate() {
diff --git a/pageserver/src/tenant/ephemeral_file.rs b/pageserver/src/tenant/ephemeral_file.rs
index 4c5fe424f3..887834cd9b 100644
--- a/pageserver/src/tenant/ephemeral_file.rs
+++ b/pageserver/src/tenant/ephemeral_file.rs
@@ -28,7 +28,7 @@ pub struct EphemeralFile {
 }
 
 impl EphemeralFile {
-    pub fn create(
+    pub async fn create(
         conf: &PageServerConf,
         tenant_id: TenantId,
         timeline_id: TimelineId,
@@ -44,7 +44,8 @@ impl EphemeralFile {
         let file = VirtualFile::open_with_options(
             &filename,
             OpenOptions::new().read(true).write(true).create(true),
-        )?;
+        )
+        .await?;
 
         Ok(EphemeralFile {
             page_cache_file_id: page_cache::next_file_id(),
@@ -286,7 +287,7 @@ mod tests {
     async fn test_ephemeral_blobs() -> Result<(), io::Error> {
         let (conf, tenant_id, timeline_id) = harness("ephemeral_blobs")?;
 
-        let mut file = EphemeralFile::create(conf, tenant_id, timeline_id)?;
+        let mut file = EphemeralFile::create(conf, tenant_id, timeline_id).await?;
 
         let pos_foo = file.write_blob(b"foo").await?;
         assert_eq!(
diff --git a/pageserver/src/tenant/par_fsync.rs b/pageserver/src/tenant/par_fsync.rs
index 3cbcfe8774..705b42aff7 100644
--- a/pageserver/src/tenant/par_fsync.rs
+++ b/pageserver/src/tenant/par_fsync.rs
@@ -4,10 +4,9 @@ use std::{
     sync::atomic::{AtomicUsize, Ordering},
 };
 
-use crate::virtual_file::VirtualFile;
-
 fn fsync_path(path: &Path) -> io::Result<()> {
-    let file = VirtualFile::open(path)?;
+    // TODO use VirtualFile::fsync_all once we fully go async.
+    let file = std::fs::File::open(path)?;
     file.sync_all()
 }
 
diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs
index b6fbf98962..6925cb59cd 100644
--- a/pageserver/src/tenant/storage_layer/delta_layer.rs
+++ b/pageserver/src/tenant/storage_layer/delta_layer.rs
@@ -604,7 +604,7 @@ impl DeltaLayerWriterInner {
         // FIXME: throw an error instead?
         let path = DeltaLayer::temp_path_for(conf, &tenant_id, &timeline_id, key_start, &lsn_range);
 
-        let mut file = VirtualFile::create(&path)?;
+        let mut file = VirtualFile::create(&path).await?;
         // make room for the header block
         file.seek(SeekFrom::Start(PAGE_SZ as u64)).await?;
         let blob_writer = BlobWriter::new(file, PAGE_SZ as u64);
@@ -732,7 +732,7 @@ impl DeltaLayerWriterInner {
         };
 
         // fsync the file
-        file.sync_all()?;
+        file.sync_all().await?;
         // Rename the file to its final name
         //
         // Note: This overwrites any existing file. There shouldn't be any.
@@ -851,6 +851,7 @@ impl DeltaLayerInner {
         summary: Option<Summary>,
     ) -> anyhow::Result<Self> {
         let file = VirtualFile::open(path)
+            .await
             .with_context(|| format!("Failed to open file '{}'", path.display()))?;
         let file = FileBlockReader::new(file);
 
diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs
index 8f7fb8175c..2a6cabcc97 100644
--- a/pageserver/src/tenant/storage_layer/image_layer.rs
+++ b/pageserver/src/tenant/storage_layer/image_layer.rs
@@ -438,6 +438,7 @@ impl ImageLayerInner {
         summary: Option<Summary>,
     ) -> anyhow::Result<Self> {
         let file = VirtualFile::open(path)
+            .await
             .with_context(|| format!("Failed to open file '{}'", path.display()))?;
         let file = FileBlockReader::new(file);
         let summary_blk = file.read_blk(0).await?;
@@ -540,7 +541,8 @@ impl ImageLayerWriterInner {
         let mut file = VirtualFile::open_with_options(
             &path,
             std::fs::OpenOptions::new().write(true).create_new(true),
-        )?;
+        )
+        .await?;
         // make room for the header block
         file.seek(SeekFrom::Start(PAGE_SZ as u64)).await?;
         let blob_writer = BlobWriter::new(file, PAGE_SZ as u64);
@@ -645,7 +647,7 @@ impl ImageLayerWriterInner {
         };
 
         // fsync the file
-        file.sync_all()?;
+        file.sync_all().await?;
 
         // Rename the file to its final name
         //
diff --git a/pageserver/src/tenant/storage_layer/inmemory_layer.rs b/pageserver/src/tenant/storage_layer/inmemory_layer.rs
index 374b0bb60c..3ff1c6bb18 100644
--- a/pageserver/src/tenant/storage_layer/inmemory_layer.rs
+++ b/pageserver/src/tenant/storage_layer/inmemory_layer.rs
@@ -236,7 +236,7 @@ impl InMemoryLayer {
     ///
     /// Create a new, empty, in-memory layer
     ///
-    pub fn create(
+    pub async fn create(
         conf: &'static PageServerConf,
         timeline_id: TimelineId,
         tenant_id: TenantId,
@@ -244,7 +244,7 @@ impl InMemoryLayer {
     ) -> Result<InMemoryLayer> {
         trace!("initializing new empty InMemoryLayer for writing on timeline {timeline_id} at {start_lsn}");
 
-        let file = EphemeralFile::create(conf, tenant_id, timeline_id)?;
+        let file = EphemeralFile::create(conf, tenant_id, timeline_id).await?;
 
         Ok(InMemoryLayer {
             conf,
diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs
index cb8b842cf6..fa3b487589 100644
--- a/pageserver/src/tenant/timeline.rs
+++ b/pageserver/src/tenant/timeline.rs
@@ -2546,13 +2546,15 @@ impl Timeline {
     ///
     async fn get_layer_for_write(&self, lsn: Lsn) -> anyhow::Result<Arc<InMemoryLayer>> {
         let mut guard = self.layers.write().await;
-        let layer = guard.get_layer_for_write(
-            lsn,
-            self.get_last_record_lsn(),
-            self.conf,
-            self.timeline_id,
-            self.tenant_id,
-        )?;
+        let layer = guard
+            .get_layer_for_write(
+                lsn,
+                self.get_last_record_lsn(),
+                self.conf,
+                self.timeline_id,
+                self.tenant_id,
+            )
+            .await?;
         Ok(layer)
     }
 
diff --git a/pageserver/src/tenant/timeline/layer_manager.rs b/pageserver/src/tenant/timeline/layer_manager.rs
index 5522ea1788..3c88d31f24 100644
--- a/pageserver/src/tenant/timeline/layer_manager.rs
+++ b/pageserver/src/tenant/timeline/layer_manager.rs
@@ -87,7 +87,7 @@ impl LayerManager {
 
     /// Open a new writable layer to append data if there is no open layer, otherwise return the current open layer,
     /// called within `get_layer_for_write`.
-    pub(crate) fn get_layer_for_write(
+    pub(crate) async fn get_layer_for_write(
         &mut self,
         lsn: Lsn,
         last_record_lsn: Lsn,
@@ -129,7 +129,7 @@ impl LayerManager {
                 lsn
             );
 
-            let new_layer = InMemoryLayer::create(conf, timeline_id, tenant_id, start_lsn)?;
+            let new_layer = InMemoryLayer::create(conf, timeline_id, tenant_id, start_lsn).await?;
             let layer = Arc::new(new_layer);
 
             self.layer_map.open_layer = Some(layer.clone());
diff --git a/pageserver/src/virtual_file.rs b/pageserver/src/virtual_file.rs
index c4241c4270..1fa5fcc297 100644
--- a/pageserver/src/virtual_file.rs
+++ b/pageserver/src/virtual_file.rs
@@ -210,17 +210,18 @@ impl CrashsafeOverwriteError {
 
 impl VirtualFile {
     /// Open a file in read-only mode. Like File::open.
-    pub fn open(path: &Path) -> Result<VirtualFile, std::io::Error> {
-        Self::open_with_options(path, OpenOptions::new().read(true))
+    pub async fn open(path: &Path) -> Result<VirtualFile, std::io::Error> {
+        Self::open_with_options(path, OpenOptions::new().read(true)).await
     }
 
     /// Create a new file for writing. If the file exists, it will be truncated.
     /// Like File::create.
-    pub fn create(path: &Path) -> Result<VirtualFile, std::io::Error> {
+    pub async fn create(path: &Path) -> Result<VirtualFile, std::io::Error> {
         Self::open_with_options(
             path,
             OpenOptions::new().write(true).create(true).truncate(true),
         )
+        .await
     }
 
     /// Open a file with given options.
@@ -228,7 +229,7 @@ impl VirtualFile {
     /// Note: If any custom flags were set in 'open_options' through OpenOptionsExt,
     /// they will be applied also when the file is subsequently re-opened, not only
     /// on the first time. Make sure that's sane!
-    pub fn open_with_options(
+    pub async fn open_with_options(
         path: &Path,
         open_options: &OpenOptions,
     ) -> Result<VirtualFile, std::io::Error> {
@@ -299,11 +300,13 @@ impl VirtualFile {
                 // we bail out instead of causing damage.
                 .create_new(true),
         )
+        .await
         .map_err(CrashsafeOverwriteError::CreateTempfile)?;
         file.write_all(content)
             .await
             .map_err(CrashsafeOverwriteError::WriteContents)?;
         file.sync_all()
+            .await
             .map_err(CrashsafeOverwriteError::SyncTempfile)?;
         drop(file); // before the rename, that's important!
                     // renames are atomic
@@ -316,26 +319,28 @@ impl VirtualFile {
         // try_lock.
         let final_parent_dirfd =
             Self::open_with_options(final_path_parent, OpenOptions::new().read(true))
+                .await
                 .map_err(CrashsafeOverwriteError::OpenFinalPathParentDir)?;
         final_parent_dirfd
             .sync_all()
+            .await
             .map_err(CrashsafeOverwriteError::SyncFinalPathParentDir)?;
         Ok(())
     }
 
     /// Call File::sync_all() on the underlying File.
-    pub fn sync_all(&self) -> Result<(), Error> {
-        self.with_file("fsync", |file| file.sync_all())?
+    pub async fn sync_all(&self) -> Result<(), Error> {
+        self.with_file("fsync", |file| file.sync_all()).await?
     }
 
     pub async fn metadata(&self) -> Result<fs::Metadata, Error> {
-        self.with_file("metadata", |file| file.metadata())?
+        self.with_file("metadata", |file| file.metadata()).await?
     }
 
     /// Helper function that looks up the underlying File for this VirtualFile,
     /// opening it and evicting some other File if necessary. It calls 'func'
     /// with the physical File.
-    fn with_file<F, R>(&self, op: &str, mut func: F) -> Result<R, Error>
+    async fn with_file<F, R>(&self, op: &str, mut func: F) -> Result<R, Error>
     where
         F: FnMut(&File) -> R,
     {
@@ -415,7 +420,9 @@ impl VirtualFile {
                 self.pos = offset;
             }
             SeekFrom::End(offset) => {
-                self.pos = self.with_file("seek", |mut file| file.seek(SeekFrom::End(offset)))??
+                self.pos = self
+                    .with_file("seek", |mut file| file.seek(SeekFrom::End(offset)))
+                    .await??
             }
             SeekFrom::Current(offset) => {
                 let pos = self.pos as i128 + offset as i128;
@@ -503,7 +510,9 @@ impl VirtualFile {
     }
 
     pub async fn read_at(&self, buf: &mut [u8], offset: u64) -> Result<usize, Error> {
-        let result = self.with_file("read", |file| file.read_at(buf, offset))?;
+        let result = self
+            .with_file("read", |file| file.read_at(buf, offset))
+            .await?;
         if let Ok(size) = result {
             STORAGE_IO_SIZE
                 .with_label_values(&["read", &self.tenant_id, &self.timeline_id])
@@ -513,7 +522,9 @@ impl VirtualFile {
     }
 
     async fn write_at(&self, buf: &[u8], offset: u64) -> Result<usize, Error> {
-        let result = self.with_file("write", |file| file.write_at(buf, offset))?;
+        let result = self
+            .with_file("write", |file| file.write_at(buf, offset))
+            .await?;
         if let Ok(size) = result {
             STORAGE_IO_SIZE
                 .with_label_values(&["write", &self.tenant_id, &self.timeline_id])
@@ -625,6 +636,7 @@ mod tests {
     use rand::seq::SliceRandom;
     use rand::thread_rng;
     use rand::Rng;
+    use std::future::Future;
     use std::io::Write;
     use std::sync::Arc;
 
@@ -694,8 +706,8 @@ mod tests {
         // results with VirtualFiles as with native Files. (Except that with
         // native files, you will run out of file descriptors if the ulimit
         // is low enough.)
-        test_files("virtual_files", |path, open_options| {
-            let vf = VirtualFile::open_with_options(path, open_options)?;
+        test_files("virtual_files", |path, open_options| async move {
+            let vf = VirtualFile::open_with_options(&path, &open_options).await?;
             Ok(MaybeVirtualFile::VirtualFile(vf))
         })
         .await
@@ -703,31 +715,37 @@ mod tests {
 
     #[tokio::test]
     async fn test_physical_files() -> Result<(), Error> {
-        test_files("physical_files", |path, open_options| {
+        test_files("physical_files", |path, open_options| async move {
             Ok(MaybeVirtualFile::File(open_options.open(path)?))
         })
         .await
     }
 
-    async fn test_files<OF>(testname: &str, openfunc: OF) -> Result<(), Error>
+    async fn test_files<OF, FT>(testname: &str, openfunc: OF) -> Result<(), Error>
     where
-        OF: Fn(&Path, &OpenOptions) -> Result<MaybeVirtualFile, std::io::Error>,
+        OF: Fn(PathBuf, OpenOptions) -> FT,
+        FT: Future<Output = Result<MaybeVirtualFile, std::io::Error>>,
     {
         let testdir = crate::config::PageServerConf::test_repo_dir(testname);
         std::fs::create_dir_all(&testdir)?;
 
         let path_a = testdir.join("file_a");
         let mut file_a = openfunc(
-            &path_a,
-            OpenOptions::new().write(true).create(true).truncate(true),
-        )?;
+            path_a.clone(),
+            OpenOptions::new()
+                .write(true)
+                .create(true)
+                .truncate(true)
+                .to_owned(),
+        )
+        .await?;
         file_a.write_all(b"foobar").await?;
 
         // cannot read from a file opened in write-only mode
         let _ = file_a.read_string().await.unwrap_err();
 
         // Close the file and re-open for reading
-        let mut file_a = openfunc(&path_a, OpenOptions::new().read(true))?;
+        let mut file_a = openfunc(path_a, OpenOptions::new().read(true).to_owned()).await?;
 
         // cannot write to a file opened in read-only mode
         let _ = file_a.write_all(b"bar").await.unwrap_err();
@@ -763,13 +781,15 @@ mod tests {
         // Create another test file, and try FileExt functions on it.
         let path_b = testdir.join("file_b");
         let mut file_b = openfunc(
-            &path_b,
+            path_b.clone(),
             OpenOptions::new()
                 .read(true)
                 .write(true)
                 .create(true)
-                .truncate(true),
-        )?;
+                .truncate(true)
+                .to_owned(),
+        )
+        .await?;
         file_b.write_all_at(b"BAR", 3).await?;
         file_b.write_all_at(b"FOO", 0).await?;
 
@@ -783,7 +803,8 @@ mod tests {
 
         let mut vfiles = Vec::new();
         for _ in 0..100 {
-            let mut vfile = openfunc(&path_b, OpenOptions::new().read(true))?;
+            let mut vfile =
+                openfunc(path_b.clone(), OpenOptions::new().read(true).to_owned()).await?;
             assert_eq!("FOOBAR", vfile.read_string().await?);
             vfiles.push(vfile);
         }
@@ -808,8 +829,8 @@ mod tests {
     /// Test using VirtualFiles from many threads concurrently. This tests both using
     /// a lot of VirtualFiles concurrently, causing evictions, and also using the same
     /// VirtualFile from multiple threads concurrently.
-    #[test]
-    fn test_vfile_concurrency() -> Result<(), Error> {
+    #[tokio::test]
+    async fn test_vfile_concurrency() -> Result<(), Error> {
         const SIZE: usize = 8 * 1024;
         const VIRTUAL_FILES: usize = 100;
         const THREADS: usize = 100;
@@ -828,7 +849,8 @@ mod tests {
         // Open the file many times.
         let mut files = Vec::new();
         for _ in 0..VIRTUAL_FILES {
-            let f = VirtualFile::open_with_options(&test_file_path, OpenOptions::new().read(true))?;
+            let f = VirtualFile::open_with_options(&test_file_path, OpenOptions::new().read(true))
+                .await?;
             files.push(f);
         }
         let files = Arc::new(files);
@@ -839,9 +861,10 @@ mod tests {
             .thread_name("test_vfile_concurrency thread")
             .build()
             .unwrap();
+        let mut hdls = Vec::new();
         for _threadno in 0..THREADS {
             let files = files.clone();
-            rt.spawn(async move {
+            let hdl = rt.spawn(async move {
                 let mut buf = [0u8; SIZE];
                 let mut rng = rand::rngs::OsRng;
                 for _ in 1..1000 {
@@ -850,7 +873,12 @@ mod tests {
                     assert!(buf == SAMPLE);
                 }
             });
+            hdls.push(hdl);
         }
+        for hdl in hdls {
+            hdl.await?;
+        }
+        std::mem::forget(rt);
 
         Ok(())
     }

From b414360afbf007c77e35b2228d6bb17ff979e7df Mon Sep 17 00:00:00 2001
From: Heikki Linnakangas <heikki@neon.tech>
Date: Fri, 8 Sep 2023 02:06:53 +0300
Subject: [PATCH 31/61] Upgrade ip4r to version 2.4.2 (#5242)

Includes PostgreSQL v16 support. No functional changes.
---
 Dockerfile.compute-node | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node
index 563a05c845..f77b274058 100644
--- a/Dockerfile.compute-node
+++ b/Dockerfile.compute-node
@@ -307,8 +307,8 @@ RUN wget https://github.com/theory/pgtap/archive/refs/tags/v1.2.0.tar.gz -O pgta
 FROM build-deps AS ip4r-pg-build
 COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
 
-RUN wget https://github.com/RhodiumToad/ip4r/archive/refs/tags/2.4.1.tar.gz -O ip4r.tar.gz && \
-    echo "78b9f0c1ae45c22182768fe892a32d533c82281035e10914111400bf6301c726 ip4r.tar.gz" | sha256sum --check && \
+RUN wget https://github.com/RhodiumToad/ip4r/archive/refs/tags/2.4.2.tar.gz -O ip4r.tar.gz && \
+    echo "0f7b1f159974f49a47842a8ab6751aecca1ed1142b6d5e38d81b064b2ead1b4b ip4r.tar.gz" | sha256sum --check && \
     mkdir ip4r-src && cd ip4r-src && tar xvzf ../ip4r.tar.gz --strip-components=1 -C . && \
     make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \
     make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \

From 252b953f18a89cea4cfda537cbec96d94f470cdf Mon Sep 17 00:00:00 2001
From: Heikki Linnakangas <heikki@neon.tech>
Date: Fri, 8 Sep 2023 02:07:17 +0300
Subject: [PATCH 32/61] Upgrade postgresql-hll to version 2.18. (#5241)

This includes PostgreSQL 16 support. No other changes, really.

The extension version in the upstream was changed from 2.17 to 2.18,
however, there is no difference between the catalog objects. So if you
had installed 2.17 previously, it will continue to work. You can run
"ALTER EXTENSION hll UPDATE", but all it will do is update the version
number in the pg_extension table.
---
 Dockerfile.compute-node | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node
index f77b274058..3e105ef1c8 100644
--- a/Dockerfile.compute-node
+++ b/Dockerfile.compute-node
@@ -339,8 +339,8 @@ RUN wget https://github.com/dimitri/prefix/archive/refs/tags/v1.2.9.tar.gz -O pr
 FROM build-deps AS hll-pg-build
 COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
 
-RUN wget https://github.com/citusdata/postgresql-hll/archive/refs/tags/v2.17.tar.gz -O hll.tar.gz && \
-    echo "9a18288e884f197196b0d29b9f178ba595b0dfc21fbf7a8699380e77fa04c1e9 hll.tar.gz" | sha256sum --check && \
+RUN wget https://github.com/citusdata/postgresql-hll/archive/refs/tags/v2.18.tar.gz -O hll.tar.gz && \
+    echo "e2f55a6f4c4ab95ee4f1b4a2b73280258c5136b161fe9d059559556079694f0e hll.tar.gz" | sha256sum --check && \
     mkdir hll-src && cd hll-src && tar xvzf ../hll.tar.gz --strip-components=1 -C . && \
     make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \
     make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \

From 6c31a2d342995a5241ba077e66b811ddbab0f3e0 Mon Sep 17 00:00:00 2001
From: Heikki Linnakangas <heikki@neon.tech>
Date: Fri, 8 Sep 2023 02:10:01 +0300
Subject: [PATCH 33/61] Upgrade prefix extension to version 1.2.10 (#5244)

This version includes trivial changes to make it compile with PostgreSQL
16. No functional changes.
---
 Dockerfile.compute-node | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node
index 3e105ef1c8..759790e01a 100644
--- a/Dockerfile.compute-node
+++ b/Dockerfile.compute-node
@@ -323,8 +323,8 @@ RUN wget https://github.com/RhodiumToad/ip4r/archive/refs/tags/2.4.2.tar.gz -O i
 FROM build-deps AS prefix-pg-build
 COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
 
-RUN wget https://github.com/dimitri/prefix/archive/refs/tags/v1.2.9.tar.gz -O prefix.tar.gz && \
-    echo "38d30a08d0241a8bbb8e1eb8f0152b385051665a8e621c8899e7c5068f8b511e prefix.tar.gz" | sha256sum --check && \
+RUN wget https://github.com/dimitri/prefix/archive/refs/tags/v1.2.10.tar.gz -O prefix.tar.gz && \
+    echo "4342f251432a5f6fb05b8597139d3ccde8dcf87e8ca1498e7ee931ca057a8575 prefix.tar.gz" | sha256sum --check && \
     mkdir prefix-src && cd prefix-src && tar xvzf ../prefix.tar.gz --strip-components=1 -C . && \
     make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \
     make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \

From 57c1ea49b3f921f7e8603d9d0282d20677ce104c Mon Sep 17 00:00:00 2001
From: Heikki Linnakangas <heikki@neon.tech>
Date: Fri, 8 Sep 2023 03:30:11 +0300
Subject: [PATCH 34/61] Update hypopg extension to version 1.4.0 (#5245)

The v1.4.0 includes changes to make it compile with PostgreSQL 16. The
commit log doesn't call it out explicitly, but I tested it manually.

v1.4.0 includes some new functions, but I tested manually that the the
v1.3.1 functionality works with the v1.4.0 version of the library. That
means that this doesn't break existing installations. Users can do
"ALTER EXTENSION hypopg UPDATE" if they want to use the new v1.4.0
functionality, but they don't have to.
---
 Dockerfile.compute-node | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node
index 759790e01a..d447ec995d 100644
--- a/Dockerfile.compute-node
+++ b/Dockerfile.compute-node
@@ -243,8 +243,8 @@ RUN wget https://github.com/michelp/pgjwt/archive/9742dab1b2f297ad3811120db7b214
 FROM build-deps AS hypopg-pg-build
 COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
 
-RUN wget https://github.com/HypoPG/hypopg/archive/refs/tags/1.3.1.tar.gz -O hypopg.tar.gz && \
-    echo "e7f01ee0259dc1713f318a108f987663d60f3041948c2ada57a94b469565ca8e hypopg.tar.gz" | sha256sum --check && \
+RUN wget https://github.com/HypoPG/hypopg/archive/refs/tags/1.4.0.tar.gz -O hypopg.tar.gz && \
+    echo "0821011743083226fc9b813c1f2ef5897a91901b57b6bea85a78e466187c6819 hypopg.tar.gz" | sha256sum --check && \
     mkdir hypopg-src && cd hypopg-src && tar xvzf ../hypopg.tar.gz --strip-components=1 -C . && \
     make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \
     make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \

From b80740bf9f644b8b35401c0d4fea4e31d3b40f86 Mon Sep 17 00:00:00 2001
From: Alexander Bayandin <alexander@neon.tech>
Date: Fri, 8 Sep 2023 01:57:38 +0100
Subject: [PATCH 35/61] test_startup: increase timeout (#5238)

## Problem

`test_runner/performance/test_startup.py::test_startup` started to fail
more frequently because of the timeout.
Let's increase the timeout to see the failures on the perf dashboard.

## Summary of changes
- Increase timeout for`test_startup` from 600 to 900 seconds
---
 test_runner/performance/test_startup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_runner/performance/test_startup.py b/test_runner/performance/test_startup.py
index fade78504a..366b9b0e68 100644
--- a/test_runner/performance/test_startup.py
+++ b/test_runner/performance/test_startup.py
@@ -84,7 +84,7 @@ def test_startup_simple(neon_env_builder: NeonEnvBuilder, zenbenchmark: NeonBenc
 
 
 # This test sometimes runs for longer than the global 5 minute timeout.
-@pytest.mark.timeout(600)
+@pytest.mark.timeout(900)
 def test_startup(neon_env_builder: NeonEnvBuilder, zenbenchmark: NeonBenchmarker):
     neon_env_builder.num_safekeepers = 3
     env = neon_env_builder.init_start()

From dac995e7e9b393f09224d3b7003b17594b72c9b9 Mon Sep 17 00:00:00 2001
From: Heikki Linnakangas <heikki@neon.tech>
Date: Fri, 8 Sep 2023 10:46:02 +0300
Subject: [PATCH 36/61] Update plpgsql_check extension to version v2.4.0
 (#5249)

This brings v16 support.
---
 Dockerfile.compute-node | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node
index d447ec995d..5786710716 100644
--- a/Dockerfile.compute-node
+++ b/Dockerfile.compute-node
@@ -355,8 +355,8 @@ RUN wget https://github.com/citusdata/postgresql-hll/archive/refs/tags/v2.18.tar
 FROM build-deps AS plpgsql-check-pg-build
 COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
 
-RUN wget https://github.com/okbob/plpgsql_check/archive/refs/tags/v2.3.2.tar.gz -O plpgsql_check.tar.gz && \
-    echo "9d81167c4bbeb74eebf7d60147b21961506161addc2aee537f95ad8efeae427b plpgsql_check.tar.gz" | sha256sum --check && \
+RUN wget https://github.com/okbob/plpgsql_check/archive/refs/tags/v2.4.0.tar.gz -O plpgsql_check.tar.gz && \
+    echo "9ba58387a279b35a3bfa39ee611e5684e6cddb2ba046ddb2c5190b3bd2ca254a plpgsql_check.tar.gz" | sha256sum --check && \
     mkdir plpgsql_check-src && cd plpgsql_check-src && tar xvzf ../plpgsql_check.tar.gz --strip-components=1 -C . && \
     make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config USE_PGXS=1 && \
     make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config USE_PGXS=1 && \

From cdc65c1857432200505f05d09d04019cbbd05531 Mon Sep 17 00:00:00 2001
From: Heikki Linnakangas <heikki@neon.tech>
Date: Fri, 8 Sep 2023 12:42:46 +0300
Subject: [PATCH 37/61] Update pg_cron to version 1.6.0 (#5252)

This includes PostgreSQL 16 support. There are no catalog changes, so
this is a drop-in replacement, no need to run "ALTER EXTENSION UPDATE".
---
 Dockerfile.compute-node | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node
index 5786710716..09380a6d0d 100644
--- a/Dockerfile.compute-node
+++ b/Dockerfile.compute-node
@@ -452,8 +452,8 @@ FROM build-deps AS pg-cron-pg-build
 COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/
 
 ENV PATH "/usr/local/pgsql/bin/:$PATH"
-RUN wget https://github.com/citusdata/pg_cron/archive/refs/tags/v1.5.2.tar.gz -O pg_cron.tar.gz && \
-    echo "6f7f0980c03f1e2a6a747060e67bf4a303ca2a50e941e2c19daeed2b44dec744 pg_cron.tar.gz" | sha256sum --check && \
+RUN wget https://github.com/citusdata/pg_cron/archive/refs/tags/v1.6.0.tar.gz -O pg_cron.tar.gz && \
+    echo "383a627867d730222c272bfd25cd5e151c578d73f696d32910c7db8c665cc7db pg_cron.tar.gz" | sha256sum --check && \
     mkdir pg_cron-src && cd pg_cron-src && tar xvzf ../pg_cron.tar.gz --strip-components=1 -C . && \
     make -j $(getconf _NPROCESSORS_ONLN) && \
     make -j $(getconf _NPROCESSORS_ONLN) install && \

From ff87fc569d71f90fa55015e910b4d9da7618a9ff Mon Sep 17 00:00:00 2001
From: Joonas Koivunen <joonas@neon.tech>
Date: Fri, 8 Sep 2023 13:54:23 +0300
Subject: [PATCH 38/61] test: Remote storage refactorings (#5243)

Remote storage cleanup split from #5198:
- pageserver, extensions, and safekeepers now have their separate remote
storage
- RemoteStorageKind has the configuration code
- S3Storage has the cleanup code
- with MOCK_S3, pageserver, extensions, safekeepers use different
buckets
- with LOCAL_FS, `repo_dir / "local_fs_remote_storage" / $user` is used
as path, where $user is `pageserver`, `safekeeper`
- no more `NeonEnvBuilder.enable_xxx_remote_storage` but one
`enable_{pageserver,extensions,safekeeper}_remote_storage`

Should not have any real changes. These will allow us to default to
`LOCAL_FS` for pageserver on the next PR, remove
`RemoteStorageKind.NOOP`, work towards #5172.

Co-authored-by: Alexander Bayandin <alexander@neon.tech>
---
 test_runner/fixtures/neon_fixtures.py         | 301 +++++------------
 test_runner/fixtures/pageserver/utils.py      |  16 +-
 test_runner/fixtures/remote_storage.py        | 304 ++++++++++++++----
 .../regress/test_attach_tenant_config.py      |  12 +-
 test_runner/regress/test_compatibility.py     |   9 +-
 .../regress/test_disk_usage_eviction.py       |   2 +-
 .../regress/test_download_extensions.py       |  33 +-
 test_runner/regress/test_gc_aggressive.py     |   4 +-
 test_runner/regress/test_import.py            |   7 +-
 test_runner/regress/test_layer_eviction.py    |   8 +-
 test_runner/regress/test_metric_collection.py |   4 +-
 test_runner/regress/test_ondemand_download.py |  24 +-
 .../regress/test_pageserver_restart.py        |   4 +-
 test_runner/regress/test_remote_storage.py    |  36 +--
 test_runner/regress/test_tenant_conf.py       |  12 +-
 test_runner/regress/test_tenant_delete.py     |  10 +-
 test_runner/regress/test_tenant_detach.py     |  36 +--
 test_runner/regress/test_tenant_relocation.py |  20 +-
 test_runner/regress/test_tenants.py           |   8 +-
 .../test_tenants_with_remote_storage.py       |  20 +-
 .../regress/test_threshold_based_eviction.py  |   2 +-
 test_runner/regress/test_timeline_delete.py   |  36 +--
 test_runner/regress/test_timeline_size.py     |  10 +-
 test_runner/regress/test_wal_acceptor.py      |  18 +-
 24 files changed, 438 insertions(+), 498 deletions(-)

diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py
index a8ca3b50a1..37524e7f5d 100644
--- a/test_runner/fixtures/neon_fixtures.py
+++ b/test_runner/fixtures/neon_fixtures.py
@@ -24,7 +24,6 @@ from urllib.parse import urlparse
 
 import asyncpg
 import backoff
-import boto3
 import jwt
 import psycopg2
 import pytest
@@ -32,7 +31,6 @@ import requests
 from _pytest.config import Config
 from _pytest.config.argparsing import Parser
 from _pytest.fixtures import FixtureRequest
-from mypy_boto3_s3 import S3Client
 
 # Type-related stuff
 from psycopg2.extensions import connection as PgConnection
@@ -47,11 +45,10 @@ from fixtures.pageserver.utils import wait_for_last_record_lsn, wait_for_upload
 from fixtures.pg_version import PgVersion
 from fixtures.port_distributor import PortDistributor
 from fixtures.remote_storage import (
-    LocalFsStorage,
     MockS3Server,
     RemoteStorage,
     RemoteStorageKind,
-    RemoteStorageUsers,
+    RemoteStorageUser,
     S3Storage,
     remote_storage_to_toml_inline_table,
 )
@@ -416,8 +413,7 @@ class NeonEnvBuilder:
         pg_version: PgVersion,
         test_name: str,
         test_output_dir: Path,
-        remote_storage: Optional[RemoteStorage] = None,
-        remote_storage_users: RemoteStorageUsers = RemoteStorageUsers.PAGESERVER,
+        pageserver_remote_storage: Optional[RemoteStorage] = None,
         pageserver_config_override: Optional[str] = None,
         num_safekeepers: int = 1,
         # Use non-standard SK ids to check for various parsing bugs
@@ -434,10 +430,14 @@ class NeonEnvBuilder:
         self.repo_dir = repo_dir
         self.rust_log_override = rust_log_override
         self.port_distributor = port_distributor
-        self.remote_storage = remote_storage
+
+        # Pageserver remote storage
+        self.pageserver_remote_storage = pageserver_remote_storage
+        # Extensions remote storage
         self.ext_remote_storage: Optional[S3Storage] = None
-        self.remote_storage_client: Optional[S3Client] = None
-        self.remote_storage_users = remote_storage_users
+        # Safekeepers remote storage
+        self.sk_remote_storage: Optional[RemoteStorage] = None
+
         self.broker = broker
         self.run_id = run_id
         self.mock_s3_server: MockS3Server = mock_s3_server
@@ -448,7 +448,6 @@ class NeonEnvBuilder:
         self.auth_enabled = auth_enabled
         self.default_branch_name = default_branch_name
         self.env: Optional[NeonEnv] = None
-        self.remote_storage_prefix: Optional[str] = None
         self.keep_remote_storage_contents: bool = True
         self.neon_binpath = neon_binpath
         self.pg_distrib_dir = pg_distrib_dir
@@ -501,150 +500,75 @@ class NeonEnvBuilder:
         the test didn't produce any invalid remote state.
         """
 
-        if not isinstance(self.remote_storage, S3Storage):
+        if not isinstance(self.pageserver_remote_storage, S3Storage):
             # The scrubber can't talk to e.g. LocalFS -- it needs
             # an HTTP endpoint (mock is fine) to connect to.
             raise RuntimeError(
-                "Cannot scrub with remote_storage={self.remote_storage}, require an S3 endpoint"
+                "Cannot scrub with remote_storage={self.pageserver_remote_storage}, require an S3 endpoint"
             )
 
         self.scrub_on_exit = True
 
-    def enable_remote_storage(
+    def enable_pageserver_remote_storage(
         self,
         remote_storage_kind: RemoteStorageKind,
-        force_enable: bool = True,
-        enable_remote_extensions: bool = False,
     ):
-        bucket_name = re.sub(r"[_\[\]]", "-", self.test_name)[:63]
+        assert self.pageserver_remote_storage is None, "remote storage is enabled already"
+        ret = self._configure_and_create_remote_storage(
+            remote_storage_kind, RemoteStorageUser.PAGESERVER
+        )
+        self.pageserver_remote_storage = ret
 
-        if remote_storage_kind == RemoteStorageKind.NOOP:
-            return
-        elif remote_storage_kind == RemoteStorageKind.LOCAL_FS:
-            self.enable_local_fs_remote_storage(force_enable=force_enable)
-        elif remote_storage_kind == RemoteStorageKind.MOCK_S3:
-            self.enable_mock_s3_remote_storage(
-                bucket_name=bucket_name,
-                force_enable=force_enable,
-                enable_remote_extensions=enable_remote_extensions,
-            )
-        elif remote_storage_kind == RemoteStorageKind.REAL_S3:
-            self.enable_real_s3_remote_storage(
-                test_name=bucket_name,
-                force_enable=force_enable,
-                enable_remote_extensions=enable_remote_extensions,
-            )
-        else:
-            raise RuntimeError(f"Unknown storage type: {remote_storage_kind}")
+    def enable_extensions_remote_storage(self, kind: RemoteStorageKind):
+        assert self.ext_remote_storage is None, "already configured extensions remote storage"
 
-        self.remote_storage_kind = remote_storage_kind
+        # there is an assumption that REAL_S3 for extensions is never
+        # cleaned up these are also special in that they have a hardcoded
+        # bucket and region, which is most likely the same as our normal
+        ext = self._configure_and_create_remote_storage(
+            kind,
+            RemoteStorageUser.EXTENSIONS,
+            bucket_name="neon-dev-extensions-eu-central-1",
+            bucket_region="eu-central-1",
+        )
+        assert isinstance(
+            ext, S3Storage
+        ), "unsure why, but only MOCK_S3 and REAL_S3 are currently supported for extensions"
+        ext.cleanup = False
+        self.ext_remote_storage = ext
 
-    def enable_local_fs_remote_storage(self, force_enable: bool = True):
-        """
-        Sets up the pageserver to use the local fs at the `test_dir/local_fs_remote_storage` path.
-        Errors, if the pageserver has some remote storage configuration already, unless `force_enable` is not set to `True`.
-        """
-        assert force_enable or self.remote_storage is None, "remote storage is enabled already"
-        self.remote_storage = LocalFsStorage(Path(self.repo_dir / "local_fs_remote_storage"))
+    def enable_safekeeper_remote_storage(self, kind: RemoteStorageKind):
+        assert self.sk_remote_storage is None, "sk_remote_storage already configured"
 
-    def enable_mock_s3_remote_storage(
+        self.sk_remote_storage = self._configure_and_create_remote_storage(
+            kind, RemoteStorageUser.SAFEKEEPER
+        )
+
+    def _configure_and_create_remote_storage(
         self,
-        bucket_name: str,
-        force_enable: bool = True,
-        enable_remote_extensions: bool = False,
-    ):
-        """
-        Sets up the pageserver to use the S3 mock server, creates the bucket, if it's not present already.
-        Starts up the mock server, if that does not run yet.
-        Errors, if the pageserver has some remote storage configuration already, unless `force_enable` is not set to `True`.
-
-        Also creates the bucket for extensions, self.ext_remote_storage bucket
-        """
-        assert force_enable or self.remote_storage is None, "remote storage is enabled already"
-        mock_endpoint = self.mock_s3_server.endpoint()
-        mock_region = self.mock_s3_server.region()
-
-        self.remote_storage_client = boto3.client(
-            "s3",
-            endpoint_url=mock_endpoint,
-            region_name=mock_region,
-            aws_access_key_id=self.mock_s3_server.access_key(),
-            aws_secret_access_key=self.mock_s3_server.secret_key(),
-        )
-        self.remote_storage_client.create_bucket(Bucket=bucket_name)
-
-        self.remote_storage = S3Storage(
+        kind: RemoteStorageKind,
+        user: RemoteStorageUser,
+        bucket_name: Optional[str] = None,
+        bucket_region: Optional[str] = None,
+    ) -> Optional[RemoteStorage]:
+        ret = kind.configure(
+            self.repo_dir,
+            self.mock_s3_server,
+            str(self.run_id),
+            self.test_name,
+            user,
             bucket_name=bucket_name,
-            endpoint=mock_endpoint,
-            bucket_region=mock_region,
-            access_key=self.mock_s3_server.access_key(),
-            secret_key=self.mock_s3_server.secret_key(),
-            prefix_in_bucket="pageserver",
+            bucket_region=bucket_region,
         )
 
-        if enable_remote_extensions:
-            self.ext_remote_storage = S3Storage(
-                bucket_name=bucket_name,
-                endpoint=mock_endpoint,
-                bucket_region=mock_region,
-                access_key=self.mock_s3_server.access_key(),
-                secret_key=self.mock_s3_server.secret_key(),
-                prefix_in_bucket="ext",
-            )
+        if kind == RemoteStorageKind.MOCK_S3:
+            assert isinstance(ret, S3Storage)
+            ret.client.create_bucket(Bucket=ret.bucket_name)
+        elif kind == RemoteStorageKind.REAL_S3:
+            assert isinstance(ret, S3Storage)
+            assert ret.cleanup, "we should not leave files in REAL_S3"
 
-    def enable_real_s3_remote_storage(
-        self,
-        test_name: str,
-        force_enable: bool = True,
-        enable_remote_extensions: bool = False,
-    ):
-        """
-        Sets up configuration to use real s3 endpoint without mock server
-        """
-        assert force_enable or self.remote_storage is None, "remote storage is enabled already"
-
-        access_key = os.getenv("AWS_ACCESS_KEY_ID")
-        assert access_key, "no aws access key provided"
-        secret_key = os.getenv("AWS_SECRET_ACCESS_KEY")
-        assert secret_key, "no aws access key provided"
-
-        # session token is needed for local runs with sso auth
-        session_token = os.getenv("AWS_SESSION_TOKEN")
-
-        bucket_name = os.getenv("REMOTE_STORAGE_S3_BUCKET")
-        assert bucket_name, "no remote storage bucket name provided"
-        region = os.getenv("REMOTE_STORAGE_S3_REGION")
-        assert region, "no remote storage region provided"
-
-        # do not leave data in real s3
-        self.keep_remote_storage_contents = False
-
-        # construct a prefix inside bucket for the particular test case and test run
-        self.remote_storage_prefix = f"{self.run_id}/{test_name}"
-
-        self.remote_storage_client = boto3.client(
-            "s3",
-            region_name=region,
-            aws_access_key_id=access_key,
-            aws_secret_access_key=secret_key,
-            aws_session_token=session_token,
-        )
-        self.remote_storage = S3Storage(
-            bucket_name=bucket_name,
-            bucket_region=region,
-            access_key=access_key,
-            secret_key=secret_key,
-            prefix_in_bucket=self.remote_storage_prefix,
-        )
-
-        if enable_remote_extensions:
-            self.ext_remote_storage = S3Storage(
-                bucket_name="neon-dev-extensions-eu-central-1",
-                bucket_region="eu-central-1",
-                access_key=access_key,
-                secret_key=secret_key,
-                prefix_in_bucket=None,
-            )
+        return ret
 
     def cleanup_local_storage(self):
         if self.preserve_database_files:
@@ -669,60 +593,10 @@ class NeonEnvBuilder:
                 directory_to_clean.rmdir()
 
     def cleanup_remote_storage(self):
-        # here wee check for true remote storage, no the local one
-        # local cleanup is not needed after test because in ci all env will be destroyed anyway
-        if self.remote_storage_prefix is None:
-            log.info("no remote storage was set up, skipping cleanup")
-            return
-
-        # Making mypy happy with allowing only `S3Storage` further.
-        # `self.remote_storage_prefix` is coupled with `S3Storage` storage type,
-        # so this line effectively a no-op
-        assert isinstance(self.remote_storage, S3Storage)
-        assert self.remote_storage_client is not None
-
-        if self.keep_remote_storage_contents:
-            log.info("keep_remote_storage_contents skipping remote storage cleanup")
-            return
-
-        log.info(
-            "removing data from test s3 bucket %s by prefix %s",
-            self.remote_storage.bucket_name,
-            self.remote_storage_prefix,
-        )
-        paginator = self.remote_storage_client.get_paginator("list_objects_v2")
-        pages = paginator.paginate(
-            Bucket=self.remote_storage.bucket_name,
-            Prefix=self.remote_storage_prefix,
-        )
-
-        # Using Any because DeleteTypeDef (from boto3-stubs) doesn't fit our case
-        objects_to_delete: Any = {"Objects": []}
-        cnt = 0
-        for item in pages.search("Contents"):
-            # weirdly when nothing is found it returns [None]
-            if item is None:
-                break
-
-            objects_to_delete["Objects"].append({"Key": item["Key"]})
-
-            # flush once aws limit reached
-            if len(objects_to_delete["Objects"]) >= 1000:
-                self.remote_storage_client.delete_objects(
-                    Bucket=self.remote_storage.bucket_name,
-                    Delete=objects_to_delete,
-                )
-                objects_to_delete = {"Objects": []}
-                cnt += 1
-
-        # flush rest
-        if len(objects_to_delete["Objects"]):
-            self.remote_storage_client.delete_objects(
-                Bucket=self.remote_storage.bucket_name,
-                Delete=objects_to_delete,
-            )
-
-        log.info(f"deleted {cnt} objects from remote storage")
+        # extensions are currently not cleaned up, disabled when creating
+        for x in [self.pageserver_remote_storage, self.ext_remote_storage, self.sk_remote_storage]:
+            if isinstance(x, S3Storage):
+                x.do_cleanup()
 
     def __enter__(self) -> "NeonEnvBuilder":
         return self
@@ -817,14 +691,13 @@ class NeonEnv:
         self.endpoints = EndpointFactory(self)
         self.safekeepers: List[Safekeeper] = []
         self.broker = config.broker
-        self.remote_storage = config.remote_storage
-        self.remote_storage_users = config.remote_storage_users
+        self.pageserver_remote_storage = config.pageserver_remote_storage
+        self.ext_remote_storage = config.ext_remote_storage
+        self.safekeepers_remote_storage = config.sk_remote_storage
         self.pg_version = config.pg_version
         self.neon_binpath = config.neon_binpath
         self.pg_distrib_dir = config.pg_distrib_dir
         self.endpoint_counter = 0
-        self.remote_storage_client = config.remote_storage_client
-        self.ext_remote_storage = config.ext_remote_storage
 
         # generate initial tenant ID here instead of letting 'neon init' generate it,
         # so that we don't need to dig it out of the config file afterwards.
@@ -907,13 +780,10 @@ class NeonEnv:
                 auth_enabled = true
                 """
                 )
-            if (
-                bool(self.remote_storage_users & RemoteStorageUsers.SAFEKEEPER)
-                and self.remote_storage is not None
-            ):
+            if config.sk_remote_storage is not None:
                 toml += textwrap.dedent(
                     f"""
-                remote_storage = "{remote_storage_to_toml_inline_table(self.remote_storage)}"
+                remote_storage = "{remote_storage_to_toml_inline_table(config.sk_remote_storage)}"
                 """
                 )
             safekeeper = Safekeeper(env=self, id=id, port=port)
@@ -1339,18 +1209,17 @@ class NeonCli(AbstractNeonCli):
 
             cmd = ["init", f"--config={tmp.name}", "--pg-version", self.env.pg_version]
 
+            storage = self.env.pageserver_remote_storage
+
             append_pageserver_param_overrides(
                 params_to_update=cmd,
-                remote_storage=self.env.remote_storage,
-                remote_storage_users=self.env.remote_storage_users,
+                remote_storage=storage,
                 pageserver_config_override=self.env.pageserver.config_override,
             )
 
             s3_env_vars = None
-            if self.env.remote_storage is not None and isinstance(
-                self.env.remote_storage, S3Storage
-            ):
-                s3_env_vars = self.env.remote_storage.access_env_vars()
+            if isinstance(storage, S3Storage):
+                s3_env_vars = storage.access_env_vars()
             res = self.raw_cli(cmd, extra_env_vars=s3_env_vars)
             res.check_returncode()
             return res
@@ -1371,15 +1240,15 @@ class NeonCli(AbstractNeonCli):
         extra_env_vars: Optional[Dict[str, str]] = None,
     ) -> "subprocess.CompletedProcess[str]":
         start_args = ["pageserver", "start", *overrides]
+        storage = self.env.pageserver_remote_storage
         append_pageserver_param_overrides(
             params_to_update=start_args,
-            remote_storage=self.env.remote_storage,
-            remote_storage_users=self.env.remote_storage_users,
+            remote_storage=storage,
             pageserver_config_override=self.env.pageserver.config_override,
         )
 
-        if self.env.remote_storage is not None and isinstance(self.env.remote_storage, S3Storage):
-            s3_env_vars = self.env.remote_storage.access_env_vars()
+        if isinstance(storage, S3Storage):
+            s3_env_vars = storage.access_env_vars()
             extra_env_vars = (extra_env_vars or {}) | s3_env_vars
 
         return self.raw_cli(start_args, extra_env_vars=extra_env_vars)
@@ -1396,8 +1265,8 @@ class NeonCli(AbstractNeonCli):
         self, id: int, extra_opts: Optional[List[str]] = None
     ) -> "subprocess.CompletedProcess[str]":
         s3_env_vars = None
-        if self.env.remote_storage is not None and isinstance(self.env.remote_storage, S3Storage):
-            s3_env_vars = self.env.remote_storage.access_env_vars()
+        if isinstance(self.env.safekeepers_remote_storage, S3Storage):
+            s3_env_vars = self.env.safekeepers_remote_storage.access_env_vars()
 
         if extra_opts is not None:
             extra_opts = [f"-e={opt}" for opt in extra_opts]
@@ -1485,9 +1354,10 @@ class NeonCli(AbstractNeonCli):
         if endpoint_id is not None:
             args.append(endpoint_id)
 
+        storage = self.env.ext_remote_storage
         s3_env_vars = None
-        if self.env.remote_storage is not None and isinstance(self.env.remote_storage, S3Storage):
-            s3_env_vars = self.env.remote_storage.access_env_vars()
+        if isinstance(storage, S3Storage):
+            s3_env_vars = storage.access_env_vars()
 
         res = self.raw_cli(args, extra_env_vars=s3_env_vars)
         res.check_returncode()
@@ -1762,10 +1632,9 @@ class NeonPageserver(PgProtocol):
 def append_pageserver_param_overrides(
     params_to_update: List[str],
     remote_storage: Optional[RemoteStorage],
-    remote_storage_users: RemoteStorageUsers,
     pageserver_config_override: Optional[str] = None,
 ):
-    if bool(remote_storage_users & RemoteStorageUsers.PAGESERVER) and remote_storage is not None:
+    if remote_storage is not None:
         remote_storage_toml_table = remote_storage_to_toml_inline_table(remote_storage)
 
         params_to_update.append(
@@ -2858,8 +2727,8 @@ class S3Scrubber:
         self.log_dir = log_dir
 
     def scrubber_cli(self, args, timeout):
-        assert isinstance(self.env.remote_storage, S3Storage)
-        s3_storage = self.env.remote_storage
+        assert isinstance(self.env.pageserver_remote_storage, S3Storage)
+        s3_storage = self.env.pageserver_remote_storage
 
         env = {
             "REGION": s3_storage.bucket_region,
diff --git a/test_runner/fixtures/pageserver/utils.py b/test_runner/fixtures/pageserver/utils.py
index 708e90d9b2..7d6dec4a14 100644
--- a/test_runner/fixtures/pageserver/utils.py
+++ b/test_runner/fixtures/pageserver/utils.py
@@ -260,15 +260,11 @@ def list_prefix(
     Note that this function takes into account prefix_in_bucket.
     """
     # For local_fs we need to properly handle empty directories, which we currently dont, so for simplicity stick to s3 api.
-    assert neon_env_builder.remote_storage_kind in (
-        RemoteStorageKind.MOCK_S3,
-        RemoteStorageKind.REAL_S3,
-    )
-    # For mypy
-    assert isinstance(neon_env_builder.remote_storage, S3Storage)
-    assert neon_env_builder.remote_storage_client is not None
+    remote = neon_env_builder.pageserver_remote_storage
+    assert isinstance(remote, S3Storage), "localfs is currently not supported"
+    assert remote.client is not None
 
-    prefix_in_bucket = neon_env_builder.remote_storage.prefix_in_bucket or ""
+    prefix_in_bucket = remote.prefix_in_bucket or ""
     if not prefix:
         prefix = prefix_in_bucket
     else:
@@ -277,9 +273,9 @@ def list_prefix(
         prefix = "/".join((prefix_in_bucket, prefix))
 
     # Note that this doesnt use pagination, so list is not guaranteed to be exhaustive.
-    response = neon_env_builder.remote_storage_client.list_objects_v2(
+    response = remote.client.list_objects_v2(
         Delimiter="/",
-        Bucket=neon_env_builder.remote_storage.bucket_name,
+        Bucket=remote.bucket_name,
         Prefix=prefix,
     )
     return response
diff --git a/test_runner/fixtures/remote_storage.py b/test_runner/fixtures/remote_storage.py
index a68257bbac..ca7ad93f83 100644
--- a/test_runner/fixtures/remote_storage.py
+++ b/test_runner/fixtures/remote_storage.py
@@ -1,10 +1,15 @@
 import enum
+import hashlib
 import json
 import os
+import re
 import subprocess
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
+
+import boto3
+from mypy_boto3_s3 import S3Client
 
 from fixtures.log_helper import log
 from fixtures.types import TenantId, TimelineId
@@ -12,6 +17,20 @@ from fixtures.types import TenantId, TimelineId
 TIMELINE_INDEX_PART_FILE_NAME = "index_part.json"
 
 
+@enum.unique
+class RemoteStorageUser(str, enum.Enum):
+    """
+    Instead of using strings for the users, use a more strict enum.
+    """
+
+    PAGESERVER = "pageserver"
+    EXTENSIONS = "ext"
+    SAFEKEEPER = "safekeeper"
+
+    def __str__(self) -> str:
+        return self.value
+
+
 class MockS3Server:
     """
     Starts a mock S3 server for testing on a port given, errors if the server fails to start or exits prematurely.
@@ -58,6 +77,124 @@ class MockS3Server:
         self.subprocess.kill()
 
 
+@dataclass
+class LocalFsStorage:
+    root: Path
+
+    def tenant_path(self, tenant_id: TenantId) -> Path:
+        return self.root / "tenants" / str(tenant_id)
+
+    def timeline_path(self, tenant_id: TenantId, timeline_id: TimelineId) -> Path:
+        return self.tenant_path(tenant_id) / "timelines" / str(timeline_id)
+
+    def index_path(self, tenant_id: TenantId, timeline_id: TimelineId) -> Path:
+        return self.timeline_path(tenant_id, timeline_id) / TIMELINE_INDEX_PART_FILE_NAME
+
+    def index_content(self, tenant_id: TenantId, timeline_id: TimelineId):
+        with self.index_path(tenant_id, timeline_id).open("r") as f:
+            return json.load(f)
+
+    def to_toml_inline_table(self) -> str:
+        return f"local_path='{self.root}'"
+
+    def cleanup(self):
+        # no cleanup is done here, because there's NeonEnvBuilder.cleanup_local_storage which will remove everything, including localfs files
+        pass
+
+    @staticmethod
+    def component_path(repo_dir: Path, user: RemoteStorageUser) -> Path:
+        return repo_dir / "local_fs_remote_storage" / str(user)
+
+
+@dataclass
+class S3Storage:
+    bucket_name: str
+    bucket_region: str
+    access_key: str
+    secret_key: str
+    prefix_in_bucket: str
+    client: S3Client
+    cleanup: bool
+    endpoint: Optional[str] = None
+
+    def access_env_vars(self) -> Dict[str, str]:
+        return {
+            "AWS_ACCESS_KEY_ID": self.access_key,
+            "AWS_SECRET_ACCESS_KEY": self.secret_key,
+        }
+
+    def to_string(self) -> str:
+        return json.dumps(
+            {
+                "bucket": self.bucket_name,
+                "region": self.bucket_region,
+                "endpoint": self.endpoint,
+                "prefix": self.prefix_in_bucket,
+            }
+        )
+
+    def to_toml_inline_table(self) -> str:
+        s = [
+            f"bucket_name='{self.bucket_name}'",
+            f"bucket_region='{self.bucket_region}'",
+        ]
+
+        if self.prefix_in_bucket is not None:
+            s.append(f"prefix_in_bucket='{self.prefix_in_bucket}'")
+
+        if self.endpoint is not None:
+            s.append(f"endpoint='{self.endpoint}'")
+
+        return ",".join(s)
+
+    def do_cleanup(self):
+        if not self.cleanup:
+            # handles previous keep_remote_storage_contents
+            return
+
+        log.info(
+            "removing data from test s3 bucket %s by prefix %s",
+            self.bucket_name,
+            self.prefix_in_bucket,
+        )
+        paginator = self.client.get_paginator("list_objects_v2")
+        pages = paginator.paginate(
+            Bucket=self.bucket_name,
+            Prefix=self.prefix_in_bucket,
+        )
+
+        # Using Any because DeleteTypeDef (from boto3-stubs) doesn't fit our case
+        objects_to_delete: Any = {"Objects": []}
+        cnt = 0
+        for item in pages.search("Contents"):
+            # weirdly when nothing is found it returns [None]
+            if item is None:
+                break
+
+            objects_to_delete["Objects"].append({"Key": item["Key"]})
+
+            # flush once aws limit reached
+            if len(objects_to_delete["Objects"]) >= 1000:
+                self.client.delete_objects(
+                    Bucket=self.bucket_name,
+                    Delete=objects_to_delete,
+                )
+                objects_to_delete = {"Objects": []}
+                cnt += 1
+
+        # flush rest
+        if len(objects_to_delete["Objects"]):
+            self.client.delete_objects(
+                Bucket=self.bucket_name,
+                Delete=objects_to_delete,
+            )
+
+        log.info(f"deleted {cnt} objects from remote storage")
+
+
+RemoteStorage = Union[LocalFsStorage, S3Storage]
+
+
 @enum.unique
 class RemoteStorageKind(str, enum.Enum):
     LOCAL_FS = "local_fs"
@@ -67,6 +204,104 @@ class RemoteStorageKind(str, enum.Enum):
     # to ensure the test pass with or without the remote storage
     NOOP = "noop"
 
+    def configure(
+        self,
+        repo_dir: Path,
+        mock_s3_server,
+        run_id: str,
+        test_name: str,
+        user: RemoteStorageUser,
+        bucket_name: Optional[str] = None,
+        bucket_region: Optional[str] = None,
+    ) -> Optional[RemoteStorage]:
+        if self == RemoteStorageKind.NOOP:
+            return None
+
+        if self == RemoteStorageKind.LOCAL_FS:
+            return LocalFsStorage(LocalFsStorage.component_path(repo_dir, user))
+
+        # real_s3 uses this as part of prefix, mock_s3 uses this as part of
+        # bucket name, giving all users unique buckets because we have to
+        # create them
+        test_name = re.sub(r"[_\[\]]", "-", test_name)
+
+        def to_bucket_name(user: str, test_name: str) -> str:
+            s = f"{user}-{test_name}"
+
+            if len(s) > 63:
+                prefix = s[:30]
+                suffix = hashlib.sha256(test_name.encode()).hexdigest()[:32]
+                s = f"{prefix}-{suffix}"
+                assert len(s) == 63
+
+            return s
+
+        if self == RemoteStorageKind.MOCK_S3:
+            # there's a single mock_s3 server for each process running the tests
+            mock_endpoint = mock_s3_server.endpoint()
+            mock_region = mock_s3_server.region()
+
+            access_key, secret_key = mock_s3_server.access_key(), mock_s3_server.secret_key()
+
+            client = boto3.client(
+                "s3",
+                endpoint_url=mock_endpoint,
+                region_name=mock_region,
+                aws_access_key_id=access_key,
+                aws_secret_access_key=secret_key,
+            )
+
+            bucket_name = to_bucket_name(user, test_name)
+            log.info(
+                f"using mock_s3 bucket name {bucket_name} for user={user}, test_name={test_name}"
+            )
+
+            return S3Storage(
+                bucket_name=bucket_name,
+                endpoint=mock_endpoint,
+                bucket_region=mock_region,
+                access_key=access_key,
+                secret_key=secret_key,
+                prefix_in_bucket="",
+                client=client,
+                cleanup=False,
+            )
+
+        assert self == RemoteStorageKind.REAL_S3
+
+        env_access_key = os.getenv("AWS_ACCESS_KEY_ID")
+        assert env_access_key, "no aws access key provided"
+        env_secret_key = os.getenv("AWS_SECRET_ACCESS_KEY")
+        assert env_secret_key, "no aws access key provided"
+
+        # session token is needed for local runs with sso auth
+        session_token = os.getenv("AWS_SESSION_TOKEN")
+
+        bucket_name = bucket_name or os.getenv("REMOTE_STORAGE_S3_BUCKET")
+        assert bucket_name is not None, "no remote storage bucket name provided"
+        bucket_region = bucket_region or os.getenv("REMOTE_STORAGE_S3_REGION")
+        assert bucket_region is not None, "no remote storage region provided"
+
+        prefix_in_bucket = f"{run_id}/{test_name}/{user}"
+
+        client = boto3.client(
+            "s3",
+            region_name=bucket_region,
+            aws_access_key_id=env_access_key,
+            aws_secret_access_key=env_secret_key,
+            aws_session_token=session_token,
+        )
+
+        return S3Storage(
+            bucket_name=bucket_name,
+            bucket_region=bucket_region,
+            access_key=env_access_key,
+            secret_key=env_secret_key,
+            prefix_in_bucket=prefix_in_bucket,
+            client=client,
+            cleanup=True,
+        )
+
 
 def available_remote_storages() -> List[RemoteStorageKind]:
     remote_storages = [RemoteStorageKind.LOCAL_FS, RemoteStorageKind.MOCK_S3]
@@ -101,72 +336,9 @@ def s3_storage() -> RemoteStorageKind:
         return RemoteStorageKind.MOCK_S3
 
 
-@dataclass
-class LocalFsStorage:
-    root: Path
-
-    def tenant_path(self, tenant_id: TenantId) -> Path:
-        return self.root / "tenants" / str(tenant_id)
-
-    def timeline_path(self, tenant_id: TenantId, timeline_id: TimelineId) -> Path:
-        return self.tenant_path(tenant_id) / "timelines" / str(timeline_id)
-
-    def index_path(self, tenant_id: TenantId, timeline_id: TimelineId) -> Path:
-        return self.timeline_path(tenant_id, timeline_id) / TIMELINE_INDEX_PART_FILE_NAME
-
-    def index_content(self, tenant_id: TenantId, timeline_id: TimelineId):
-        with self.index_path(tenant_id, timeline_id).open("r") as f:
-            return json.load(f)
-
-
-@dataclass
-class S3Storage:
-    bucket_name: str
-    bucket_region: str
-    access_key: str
-    secret_key: str
-    endpoint: Optional[str] = None
-    prefix_in_bucket: Optional[str] = ""
-
-    def access_env_vars(self) -> Dict[str, str]:
-        return {
-            "AWS_ACCESS_KEY_ID": self.access_key,
-            "AWS_SECRET_ACCESS_KEY": self.secret_key,
-        }
-
-    def to_string(self) -> str:
-        return json.dumps(
-            {
-                "bucket": self.bucket_name,
-                "region": self.bucket_region,
-                "endpoint": self.endpoint,
-                "prefix": self.prefix_in_bucket,
-            }
-        )
-
-
-RemoteStorage = Union[LocalFsStorage, S3Storage]
-
-
 # serialize as toml inline table
 def remote_storage_to_toml_inline_table(remote_storage: RemoteStorage) -> str:
-    if isinstance(remote_storage, LocalFsStorage):
-        remote_storage_config = f"local_path='{remote_storage.root}'"
-    elif isinstance(remote_storage, S3Storage):
-        remote_storage_config = f"bucket_name='{remote_storage.bucket_name}',\
-            bucket_region='{remote_storage.bucket_region}'"
-
-        if remote_storage.prefix_in_bucket is not None:
-            remote_storage_config += f",prefix_in_bucket='{remote_storage.prefix_in_bucket}'"
-
-        if remote_storage.endpoint is not None:
-            remote_storage_config += f",endpoint='{remote_storage.endpoint}'"
-    else:
+    if not isinstance(remote_storage, (LocalFsStorage, S3Storage)):
         raise Exception("invalid remote storage type")
 
-    return f"{{{remote_storage_config}}}"
-
-
-class RemoteStorageUsers(enum.Flag):
-    PAGESERVER = enum.auto()
-    SAFEKEEPER = enum.auto()
+    return f"{{{remote_storage.to_toml_inline_table()}}}"
diff --git a/test_runner/regress/test_attach_tenant_config.py b/test_runner/regress/test_attach_tenant_config.py
index 1acf429c52..892e40b19b 100644
--- a/test_runner/regress/test_attach_tenant_config.py
+++ b/test_runner/regress/test_attach_tenant_config.py
@@ -14,16 +14,14 @@ from fixtures.utils import wait_until
 
 @pytest.fixture
 def positive_env(neon_env_builder: NeonEnvBuilder) -> NeonEnv:
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=RemoteStorageKind.LOCAL_FS,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS)
     env = neon_env_builder.init_start()
 
     # eviction might be the first one after an attach to access the layers
     env.pageserver.allowed_errors.append(
         ".*unexpectedly on-demand downloading remote layer remote.* for task kind Eviction"
     )
-    assert isinstance(env.remote_storage, LocalFsStorage)
+    assert isinstance(env.pageserver_remote_storage, LocalFsStorage)
     return env
 
 
@@ -36,11 +34,9 @@ class NegativeTests:
 
 @pytest.fixture
 def negative_env(neon_env_builder: NeonEnvBuilder) -> Generator[NegativeTests, None, None]:
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=RemoteStorageKind.LOCAL_FS,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS)
     env = neon_env_builder.init_start()
-    assert isinstance(env.remote_storage, LocalFsStorage)
+    assert isinstance(env.pageserver_remote_storage, LocalFsStorage)
 
     ps_http = env.pageserver.http_client()
     (tenant_id, _) = env.neon_cli.create_tenant()
diff --git a/test_runner/regress/test_compatibility.py b/test_runner/regress/test_compatibility.py
index 6979b99708..247b1d8bce 100644
--- a/test_runner/regress/test_compatibility.py
+++ b/test_runner/regress/test_compatibility.py
@@ -22,6 +22,7 @@ from fixtures.pageserver.utils import (
 )
 from fixtures.pg_version import PgVersion
 from fixtures.port_distributor import PortDistributor
+from fixtures.remote_storage import LocalFsStorage, RemoteStorageKind, RemoteStorageUser
 from fixtures.types import Lsn
 from pytest import FixtureRequest
 
@@ -61,7 +62,7 @@ def test_create_snapshot(
     # There's no cleanup here, it allows to adjust the data in `test_backward_compatibility` itself without re-collecting it.
     neon_env_builder.pg_version = pg_version
     neon_env_builder.num_safekeepers = 3
-    neon_env_builder.enable_local_fs_remote_storage()
+    neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS)
 
     env = neon_env_builder.init_start()
     endpoint = env.endpoints.create_start("main")
@@ -260,7 +261,9 @@ def prepare_snapshot(
     # Update paths and ports in config files
     pageserver_toml = repo_dir / "pageserver.toml"
     pageserver_config = toml.load(pageserver_toml)
-    pageserver_config["remote_storage"]["local_path"] = str(repo_dir / "local_fs_remote_storage")
+    pageserver_config["remote_storage"]["local_path"] = LocalFsStorage.component_path(
+        repo_dir, RemoteStorageUser.PAGESERVER
+    )
     for param in ("listen_http_addr", "listen_pg_addr", "broker_endpoint"):
         pageserver_config[param] = port_distributor.replace_with_new_port(pageserver_config[param])
 
@@ -338,6 +341,8 @@ def check_neon_works(
     config.initial_tenant = snapshot_config["default_tenant_id"]
     config.pg_distrib_dir = pg_distrib_dir
     config.remote_storage = None
+    config.ext_remote_storage = None
+    config.sk_remote_storage = None
 
     # Use the "target" binaries to launch the storage nodes
     config_target = config
diff --git a/test_runner/regress/test_disk_usage_eviction.py b/test_runner/regress/test_disk_usage_eviction.py
index cdbd02de03..32eb910188 100644
--- a/test_runner/regress/test_disk_usage_eviction.py
+++ b/test_runner/regress/test_disk_usage_eviction.py
@@ -135,7 +135,7 @@ def eviction_env(request, neon_env_builder: NeonEnvBuilder, pg_bin: PgBin) -> Ev
 
     log.info(f"setting up eviction_env for test {request.node.name}")
 
-    neon_env_builder.enable_remote_storage(RemoteStorageKind.LOCAL_FS)
+    neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS)
 
     # initial tenant will not be present on this pageserver
     env = neon_env_builder.init_configs()
diff --git a/test_runner/regress/test_download_extensions.py b/test_runner/regress/test_download_extensions.py
index 54f51414bb..5f5c733b76 100644
--- a/test_runner/regress/test_download_extensions.py
+++ b/test_runner/regress/test_download_extensions.py
@@ -10,7 +10,11 @@ from fixtures.neon_fixtures import (
     NeonEnvBuilder,
 )
 from fixtures.pg_version import PgVersion
-from fixtures.remote_storage import RemoteStorageKind, available_s3_storages
+from fixtures.remote_storage import (
+    RemoteStorageKind,
+    S3Storage,
+    available_s3_storages,
+)
 
 
 # Cleaning up downloaded files is important for local tests
@@ -72,7 +76,8 @@ def upload_files(env):
 
             with open(full_path, "rb") as f:
                 log.info(f"UPLOAD {full_path} to ext/{full_path}")
-                env.remote_storage_client.upload_fileobj(
+                assert isinstance(env.pageserver_remote_storage, S3Storage)
+                env.pageserver_remote_storage.client.upload_fileobj(
                     f,
                     env.ext_remote_storage.bucket_name,
                     f"ext/{full_path}",
@@ -88,16 +93,12 @@ def test_remote_extensions(
     remote_storage_kind: RemoteStorageKind,
     pg_version: PgVersion,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-        enable_remote_extensions=True,
-    )
+    neon_env_builder.enable_extensions_remote_storage(remote_storage_kind)
     env = neon_env_builder.init_start()
     tenant_id, _ = env.neon_cli.create_tenant()
     env.neon_cli.create_timeline("test_remote_extensions", tenant_id=tenant_id)
 
     assert env.ext_remote_storage is not None  # satisfy mypy
-    assert env.remote_storage_client is not None  # satisfy mypy
 
     # For MOCK_S3 we upload test files.
     # For REAL_S3 we use the files already in the bucket
@@ -154,16 +155,12 @@ def test_remote_library(
     remote_storage_kind: RemoteStorageKind,
     pg_version: PgVersion,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-        enable_remote_extensions=True,
-    )
+    neon_env_builder.enable_extensions_remote_storage(remote_storage_kind)
     env = neon_env_builder.init_start()
     tenant_id, _ = env.neon_cli.create_tenant()
     env.neon_cli.create_timeline("test_remote_library", tenant_id=tenant_id)
 
     assert env.ext_remote_storage is not None  # satisfy mypy
-    assert env.remote_storage_client is not None  # satisfy mypy
 
     # For MOCK_S3 we upload test files.
     # For REAL_S3 we use the files already in the bucket
@@ -214,16 +211,12 @@ def test_multiple_extensions_one_archive(
     neon_env_builder: NeonEnvBuilder,
     pg_version: PgVersion,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=RemoteStorageKind.REAL_S3,
-        enable_remote_extensions=True,
-    )
+    neon_env_builder.enable_extensions_remote_storage(RemoteStorageKind.REAL_S3)
     env = neon_env_builder.init_start()
     tenant_id, _ = env.neon_cli.create_tenant()
     env.neon_cli.create_timeline("test_multiple_extensions_one_archive", tenant_id=tenant_id)
 
     assert env.ext_remote_storage is not None  # satisfy mypy
-    assert env.remote_storage_client is not None  # satisfy mypy
 
     endpoint = env.endpoints.create_start(
         "test_multiple_extensions_one_archive",
@@ -261,16 +254,12 @@ def test_extension_download_after_restart(
     if "15" in pg_version:  # SKIP v15 for now because test set only has extension built for v14
         return None
 
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=RemoteStorageKind.MOCK_S3,
-        enable_remote_extensions=True,
-    )
+    neon_env_builder.enable_extensions_remote_storage(RemoteStorageKind.MOCK_S3)
     env = neon_env_builder.init_start()
     tenant_id, _ = env.neon_cli.create_tenant()
     env.neon_cli.create_timeline("test_extension_download_after_restart", tenant_id=tenant_id)
 
     assert env.ext_remote_storage is not None  # satisfy mypy
-    assert env.remote_storage_client is not None  # satisfy mypy
 
     # For MOCK_S3 we upload test files.
     upload_files(env)
diff --git a/test_runner/regress/test_gc_aggressive.py b/test_runner/regress/test_gc_aggressive.py
index 8d7a42a805..db0020a8a4 100644
--- a/test_runner/regress/test_gc_aggressive.py
+++ b/test_runner/regress/test_gc_aggressive.py
@@ -100,9 +100,7 @@ def test_gc_index_upload(neon_env_builder: NeonEnvBuilder, remote_storage_kind:
     # Disable time-based pitr, we will use LSN-based thresholds in the manual GC calls
     neon_env_builder.pageserver_config_override = "tenant_config={pitr_interval = '0 sec'}"
 
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start()
     tenant_id = env.initial_tenant
diff --git a/test_runner/regress/test_import.py b/test_runner/regress/test_import.py
index 68086d3f84..ec1196eea6 100644
--- a/test_runner/regress/test_import.py
+++ b/test_runner/regress/test_import.py
@@ -19,6 +19,7 @@ from fixtures.pageserver.utils import (
     wait_for_last_record_lsn,
     wait_for_upload,
 )
+from fixtures.remote_storage import RemoteStorageKind
 from fixtures.types import Lsn, TenantId, TimelineId
 from fixtures.utils import subprocess_capture
 
@@ -80,7 +81,7 @@ def test_import_from_vanilla(test_output_dir, pg_bin, vanilla_pg, neon_env_build
     timeline = TimelineId.generate()
 
     # Set up pageserver for import
-    neon_env_builder.enable_local_fs_remote_storage()
+    neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS)
     env = neon_env_builder.init_start()
 
     client = env.pageserver.http_client()
@@ -163,7 +164,7 @@ def test_import_from_vanilla(test_output_dir, pg_bin, vanilla_pg, neon_env_build
 
 
 def test_import_from_pageserver_small(pg_bin: PgBin, neon_env_builder: NeonEnvBuilder):
-    neon_env_builder.enable_local_fs_remote_storage()
+    neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS)
     env = neon_env_builder.init_start()
 
     # FIXME: Is this expected?
@@ -185,7 +186,7 @@ def test_import_from_pageserver_small(pg_bin: PgBin, neon_env_builder: NeonEnvBu
 # @pytest.mark.skipif(os.environ.get('BUILD_TYPE') == "debug", reason="only run with release build")
 @pytest.mark.skip("See https://github.com/neondatabase/neon/issues/2255")
 def test_import_from_pageserver_multisegment(pg_bin: PgBin, neon_env_builder: NeonEnvBuilder):
-    neon_env_builder.enable_local_fs_remote_storage()
+    neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS)
     env = neon_env_builder.init_start()
 
     timeline = env.neon_cli.create_branch("test_import_from_pageserver_multisegment")
diff --git a/test_runner/regress/test_layer_eviction.py b/test_runner/regress/test_layer_eviction.py
index c939ace803..5bb953ffb0 100644
--- a/test_runner/regress/test_layer_eviction.py
+++ b/test_runner/regress/test_layer_eviction.py
@@ -19,9 +19,7 @@ def test_basic_eviction(
     neon_env_builder: NeonEnvBuilder,
     remote_storage_kind: RemoteStorageKind,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start(
         initial_tenant_conf={
@@ -154,9 +152,7 @@ def test_basic_eviction(
 
 
 def test_gc_of_remote_layers(neon_env_builder: NeonEnvBuilder):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=RemoteStorageKind.LOCAL_FS,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS)
 
     env = neon_env_builder.init_start()
 
diff --git a/test_runner/regress/test_metric_collection.py b/test_runner/regress/test_metric_collection.py
index e4641cff05..9ec649ee24 100644
--- a/test_runner/regress/test_metric_collection.py
+++ b/test_runner/regress/test_metric_collection.py
@@ -94,9 +94,7 @@ def test_metric_collection(
         + "tenant_config={pitr_interval = '0 sec'}"
     )
 
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     log.info(f"test_metric_collection endpoint is {metric_collection_endpoint}")
 
diff --git a/test_runner/regress/test_ondemand_download.py b/test_runner/regress/test_ondemand_download.py
index 0ca6a7a595..4d5971cd11 100644
--- a/test_runner/regress/test_ondemand_download.py
+++ b/test_runner/regress/test_ondemand_download.py
@@ -52,9 +52,7 @@ def test_ondemand_download_large_rel(
     neon_env_builder: NeonEnvBuilder,
     remote_storage_kind: RemoteStorageKind,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     # thinking about using a shared environment? the test assumes that global
     # metrics are for single tenant.
@@ -154,9 +152,7 @@ def test_ondemand_download_timetravel(
     neon_env_builder: NeonEnvBuilder,
     remote_storage_kind: RemoteStorageKind,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     # thinking about using a shared environment? the test assumes that global
     # metrics are for single tenant.
@@ -315,9 +311,7 @@ def test_download_remote_layers_api(
     neon_env_builder: NeonEnvBuilder,
     remote_storage_kind: RemoteStorageKind,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     ##### First start, insert data and upload it to the remote storage
     env = neon_env_builder.init_start(
@@ -476,9 +470,7 @@ def test_compaction_downloads_on_demand_without_image_creation(
     """
     Create a few layers, then evict, then make sure compaction runs successfully.
     """
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     conf = {
         # Disable background GC & compaction
@@ -563,9 +555,7 @@ def test_compaction_downloads_on_demand_with_image_creation(
     Due to current implementation, this will make image creation on-demand download layers, but we cannot really
     directly test for it.
     """
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     conf = {
         # Disable background GC & compaction
@@ -663,9 +653,7 @@ def test_ondemand_download_failure_to_replace(
     See: https://github.com/neondatabase/neon/issues/3533
     """
 
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     # disable gc and compaction via default tenant config because config is lost while detaching
     # so that compaction will not be the one to download the layer but the http handler is
diff --git a/test_runner/regress/test_pageserver_restart.py b/test_runner/regress/test_pageserver_restart.py
index ad625df1cc..2965a354bd 100644
--- a/test_runner/regress/test_pageserver_restart.py
+++ b/test_runner/regress/test_pageserver_restart.py
@@ -11,7 +11,7 @@ from fixtures.remote_storage import s3_storage
 @pytest.mark.parametrize("generations", [True, False])
 def test_pageserver_restart(neon_env_builder: NeonEnvBuilder, generations: bool):
     neon_env_builder.enable_generations = generations
-    neon_env_builder.enable_remote_storage(remote_storage_kind=s3_storage())
+    neon_env_builder.enable_pageserver_remote_storage(s3_storage())
     neon_env_builder.enable_scrub_on_exit()
 
     env = neon_env_builder.init_start()
@@ -115,7 +115,7 @@ def test_pageserver_restart(neon_env_builder: NeonEnvBuilder, generations: bool)
 # safekeeper and compute node keep running.
 @pytest.mark.timeout(540)
 def test_pageserver_chaos(neon_env_builder: NeonEnvBuilder):
-    neon_env_builder.enable_remote_storage(remote_storage_kind=s3_storage())
+    neon_env_builder.enable_pageserver_remote_storage(s3_storage())
     neon_env_builder.enable_scrub_on_exit()
 
     env = neon_env_builder.init_start()
diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py
index 3d9655cb87..b01c4cc332 100644
--- a/test_runner/regress/test_remote_storage.py
+++ b/test_runner/regress/test_remote_storage.py
@@ -60,9 +60,7 @@ def test_remote_storage_backup_and_restore(
     # and this test needs SK to write data to pageserver, so it will be visible
     neon_env_builder.safekeepers_id_start = 12
 
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     neon_env_builder.enable_generations = generations
 
@@ -224,9 +222,7 @@ def test_remote_storage_upload_queue_retries(
     neon_env_builder: NeonEnvBuilder,
     remote_storage_kind: RemoteStorageKind,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start()
 
@@ -379,9 +375,7 @@ def test_remote_timeline_client_calls_started_metric(
     neon_env_builder: NeonEnvBuilder,
     remote_storage_kind: RemoteStorageKind,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     # thinking about using a shared environment? the test assumes that global
     # metrics are for single tenant.
@@ -521,9 +515,7 @@ def test_timeline_deletion_with_files_stuck_in_upload_queue(
     neon_env_builder: NeonEnvBuilder,
     remote_storage_kind: RemoteStorageKind,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start(
         initial_tenant_conf={
@@ -612,8 +604,8 @@ def test_timeline_deletion_with_files_stuck_in_upload_queue(
     assert not timeline_path.exists()
 
     # to please mypy
-    assert isinstance(env.remote_storage, LocalFsStorage)
-    remote_timeline_path = env.remote_storage.timeline_path(tenant_id, timeline_id)
+    assert isinstance(env.pageserver_remote_storage, LocalFsStorage)
+    remote_timeline_path = env.pageserver_remote_storage.timeline_path(tenant_id, timeline_id)
 
     assert not list(remote_timeline_path.iterdir())
 
@@ -638,9 +630,7 @@ def test_empty_branch_remote_storage_upload(
     neon_env_builder: NeonEnvBuilder,
     remote_storage_kind: RemoteStorageKind,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start()
     client = env.pageserver.http_client()
@@ -689,9 +679,7 @@ def test_empty_branch_remote_storage_upload_on_restart(
     — the upload should be scheduled by load, and create_timeline should await
     for it even though it gets 409 Conflict.
     """
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start()
     client = env.pageserver.http_client()
@@ -719,9 +707,9 @@ def test_empty_branch_remote_storage_upload_on_restart(
     local_metadata = env.timeline_dir(env.initial_tenant, new_branch_timeline_id) / "metadata"
     assert local_metadata.is_file()
 
-    assert isinstance(env.remote_storage, LocalFsStorage)
+    assert isinstance(env.pageserver_remote_storage, LocalFsStorage)
 
-    new_branch_on_remote_storage = env.remote_storage.timeline_path(
+    new_branch_on_remote_storage = env.pageserver_remote_storage.timeline_path(
         env.initial_tenant, new_branch_timeline_id
     )
     assert (
@@ -786,9 +774,7 @@ def test_compaction_delete_before_upload(
     neon_env_builder: NeonEnvBuilder,
     remote_storage_kind: RemoteStorageKind,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start(
         initial_tenant_conf={
diff --git a/test_runner/regress/test_tenant_conf.py b/test_runner/regress/test_tenant_conf.py
index 93ba5477a6..4fe6909433 100644
--- a/test_runner/regress/test_tenant_conf.py
+++ b/test_runner/regress/test_tenant_conf.py
@@ -292,12 +292,10 @@ eviction_policy = { "kind" = "LayerAccessThreshold", period = "20s", threshold =
 
 
 def test_creating_tenant_conf_after_attach(neon_env_builder: NeonEnvBuilder):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=RemoteStorageKind.LOCAL_FS,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS)
 
     env = neon_env_builder.init_start()
-    assert isinstance(env.remote_storage, LocalFsStorage)
+    assert isinstance(env.pageserver_remote_storage, LocalFsStorage)
 
     # tenant is created with defaults, as in without config file
     (tenant_id, timeline_id) = env.neon_cli.create_tenant()
@@ -336,12 +334,10 @@ def test_creating_tenant_conf_after_attach(neon_env_builder: NeonEnvBuilder):
 def test_live_reconfig_get_evictions_low_residence_duration_metric_threshold(
     neon_env_builder: NeonEnvBuilder,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=RemoteStorageKind.LOCAL_FS,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS)
 
     env = neon_env_builder.init_start()
-    assert isinstance(env.remote_storage, LocalFsStorage)
+    assert isinstance(env.pageserver_remote_storage, LocalFsStorage)
 
     (tenant_id, timeline_id) = env.neon_cli.create_tenant()
     ps_http = env.pageserver.http_client()
diff --git a/test_runner/regress/test_tenant_delete.py b/test_runner/regress/test_tenant_delete.py
index c7324cac83..68188eb80a 100644
--- a/test_runner/regress/test_tenant_delete.py
+++ b/test_runner/regress/test_tenant_delete.py
@@ -41,9 +41,7 @@ def test_tenant_delete_smoke(
 ):
     neon_env_builder.pageserver_config_override = "test_remote_failures=1"
 
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start()
 
@@ -176,7 +174,7 @@ def test_delete_tenant_exercise_crash_safety_failpoints(
     if simulate_failures:
         neon_env_builder.pageserver_config_override = "test_remote_failures=1"
 
-    neon_env_builder.enable_remote_storage(remote_storage_kind)
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start(initial_tenant_conf=MANY_SMALL_LAYERS_TENANT_CONFIG)
 
@@ -295,9 +293,7 @@ def test_tenant_delete_is_resumed_on_attach(
     remote_storage_kind: RemoteStorageKind,
     pg_bin: PgBin,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start(initial_tenant_conf=MANY_SMALL_LAYERS_TENANT_CONFIG)
 
diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py
index 20526cd0a9..4e3c13123d 100644
--- a/test_runner/regress/test_tenant_detach.py
+++ b/test_runner/regress/test_tenant_detach.py
@@ -44,9 +44,7 @@ def test_tenant_reattach(
     neon_env_builder: NeonEnvBuilder,
     remote_storage_kind: RemoteStorageKind,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     # Exercise retry code path by making all uploads and downloads fail for the
     # first time. The retries print INFO-messages to the log; we will check
@@ -228,9 +226,7 @@ def test_tenant_reattach_while_busy(
     neon_env_builder: NeonEnvBuilder,
     remote_storage_kind: RemoteStorageKind,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
     env = neon_env_builder.init_start()
 
     pageserver_http = env.pageserver.http_client()
@@ -449,9 +445,7 @@ def test_detach_while_attaching(
     neon_env_builder: NeonEnvBuilder,
     remote_storage_kind: RemoteStorageKind,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     ##### First start, insert secret data and upload it to the remote storage
     env = neon_env_builder.init_start()
@@ -532,9 +526,7 @@ def test_detach_while_attaching(
 def test_ignored_tenant_reattach(
     neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
     env = neon_env_builder.init_start()
     pageserver_http = env.pageserver.http_client()
 
@@ -603,9 +595,7 @@ def test_ignored_tenant_reattach(
 def test_ignored_tenant_download_missing_layers(
     neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
     env = neon_env_builder.init_start()
     pageserver_http = env.pageserver.http_client()
     endpoint = env.endpoints.create_start("main")
@@ -668,9 +658,7 @@ def test_ignored_tenant_download_missing_layers(
 def test_ignored_tenant_stays_broken_without_metadata(
     neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
     env = neon_env_builder.init_start()
     pageserver_http = env.pageserver.http_client()
     env.endpoints.create_start("main")
@@ -711,9 +699,7 @@ def test_ignored_tenant_stays_broken_without_metadata(
 def test_load_attach_negatives(
     neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
     env = neon_env_builder.init_start()
     pageserver_http = env.pageserver.http_client()
     env.endpoints.create_start("main")
@@ -755,9 +741,7 @@ def test_ignore_while_attaching(
     neon_env_builder: NeonEnvBuilder,
     remote_storage_kind: RemoteStorageKind,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start()
     pageserver_http = env.pageserver.http_client()
@@ -858,9 +842,7 @@ def test_metrics_while_ignoring_broken_tenant_and_reloading(
     neon_env_builder: NeonEnvBuilder,
     remote_storage_kind: RemoteStorageKind,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start()
 
diff --git a/test_runner/regress/test_tenant_relocation.py b/test_runner/regress/test_tenant_relocation.py
index 6a81ff498e..88f921c2cd 100644
--- a/test_runner/regress/test_tenant_relocation.py
+++ b/test_runner/regress/test_tenant_relocation.py
@@ -22,7 +22,11 @@ from fixtures.pageserver.utils import (
     wait_tenant_status_404,
 )
 from fixtures.port_distributor import PortDistributor
-from fixtures.remote_storage import RemoteStorageKind, available_remote_storages
+from fixtures.remote_storage import (
+    LocalFsStorage,
+    RemoteStorageKind,
+    available_remote_storages,
+)
 from fixtures.types import Lsn, TenantId, TimelineId
 from fixtures.utils import (
     query_scalar,
@@ -264,7 +268,7 @@ def test_tenant_relocation(
     method: str,
     with_load: str,
 ):
-    neon_env_builder.enable_local_fs_remote_storage()
+    neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS)
 
     env = neon_env_builder.init_start()
 
@@ -278,8 +282,8 @@ def test_tenant_relocation(
     # Needed for detach polling.
     env.pageserver.allowed_errors.append(f".*NotFound: tenant {tenant_id}.*")
 
-    # create folder for remote storage mock
-    remote_storage_mock_path = env.repo_dir / "local_fs_remote_storage"
+    assert isinstance(env.pageserver_remote_storage, LocalFsStorage)
+    remote_storage_mock_path = env.pageserver_remote_storage.root
 
     # we use two branches to check that they are both relocated
     # first branch is used for load, compute for second one is used to
@@ -524,9 +528,7 @@ def test_emergency_relocate_with_branches_slow_replay(
     neon_env_builder: NeonEnvBuilder,
     remote_storage_kind: RemoteStorageKind,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start()
     env.pageserver.is_testing_enabled_or_skip()
@@ -680,9 +682,7 @@ def test_emergency_relocate_with_branches_createdb(
     neon_env_builder: NeonEnvBuilder,
     remote_storage_kind: RemoteStorageKind,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start()
     pageserver_http = env.pageserver.http_client()
diff --git a/test_runner/regress/test_tenants.py b/test_runner/regress/test_tenants.py
index 985bd63b24..88094b8766 100644
--- a/test_runner/regress/test_tenants.py
+++ b/test_runner/regress/test_tenants.py
@@ -242,9 +242,7 @@ def test_pageserver_metrics_removed_after_detach(
 ):
     """Tests that when a tenant is detached, the tenant specific metrics are not left behind"""
 
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     neon_env_builder.num_safekeepers = 3
 
@@ -302,9 +300,7 @@ def test_pageserver_metrics_removed_after_detach(
 def test_pageserver_with_empty_tenants(
     neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start()
 
diff --git a/test_runner/regress/test_tenants_with_remote_storage.py b/test_runner/regress/test_tenants_with_remote_storage.py
index 6a541c8a37..bad090e83e 100644
--- a/test_runner/regress/test_tenants_with_remote_storage.py
+++ b/test_runner/regress/test_tenants_with_remote_storage.py
@@ -62,9 +62,7 @@ async def all_tenants_workload(env: NeonEnv, tenants_endpoints):
 
 @pytest.mark.parametrize("remote_storage_kind", available_remote_storages())
 def test_tenants_many(neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start()
 
@@ -114,9 +112,7 @@ def test_tenants_many(neon_env_builder: NeonEnvBuilder, remote_storage_kind: Rem
 def test_tenants_attached_after_download(
     neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     data_id = 1
     data_secret = "very secret secret"
@@ -228,13 +224,11 @@ def test_tenant_redownloads_truncated_file_on_startup(
     remote_storage_kind = RemoteStorageKind.LOCAL_FS
 
     # since we now store the layer file length metadata, we notice on startup that a layer file is of wrong size, and proceed to redownload it.
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start()
 
-    assert isinstance(env.remote_storage, LocalFsStorage)
+    assert isinstance(env.pageserver_remote_storage, LocalFsStorage)
 
     env.pageserver.allowed_errors.append(
         ".*removing local file .* because it has unexpected length.*"
@@ -278,7 +272,7 @@ def test_tenant_redownloads_truncated_file_on_startup(
     (path, expected_size) = local_layer_truncated
 
     # ensure the same size is found from the index_part.json
-    index_part = env.remote_storage.index_content(tenant_id, timeline_id)
+    index_part = env.pageserver_remote_storage.index_content(tenant_id, timeline_id)
     assert index_part["layer_metadata"][path.name]["file_size"] == expected_size
 
     ## Start the pageserver. It will notice that the file size doesn't match, and
@@ -308,7 +302,9 @@ def test_tenant_redownloads_truncated_file_on_startup(
     assert os.stat(path).st_size == expected_size, "truncated layer should had been re-downloaded"
 
     # the remote side of local_layer_truncated
-    remote_layer_path = env.remote_storage.timeline_path(tenant_id, timeline_id) / path.name
+    remote_layer_path = (
+        env.pageserver_remote_storage.timeline_path(tenant_id, timeline_id) / path.name
+    )
 
     # if the upload ever was ongoing, this check would be racy, but at least one
     # extra http request has been made in between so assume it's enough delay
diff --git a/test_runner/regress/test_threshold_based_eviction.py b/test_runner/regress/test_threshold_based_eviction.py
index 739a9a5b74..6f5e3e2518 100644
--- a/test_runner/regress/test_threshold_based_eviction.py
+++ b/test_runner/regress/test_threshold_based_eviction.py
@@ -21,7 +21,7 @@ def test_threshold_based_eviction(
     pg_bin: PgBin,
     neon_env_builder: NeonEnvBuilder,
 ):
-    neon_env_builder.enable_remote_storage(RemoteStorageKind.LOCAL_FS)
+    neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS)
 
     # Start with metrics collection enabled, so that the eviction task
     # imitates its accesses. We'll use a non-existent endpoint to make it fail.
diff --git a/test_runner/regress/test_timeline_delete.py b/test_runner/regress/test_timeline_delete.py
index 0ce714d185..cc770e5aa8 100644
--- a/test_runner/regress/test_timeline_delete.py
+++ b/test_runner/regress/test_timeline_delete.py
@@ -191,7 +191,7 @@ def test_delete_timeline_exercise_crash_safety_failpoints(
     8. Retry or restart without the failpoint and check the result.
     """
 
-    neon_env_builder.enable_remote_storage(remote_storage_kind)
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start(
         initial_tenant_conf={
@@ -346,9 +346,7 @@ def test_timeline_resurrection_on_attach(
     Original issue: https://github.com/neondatabase/neon/issues/3560
     """
 
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     ##### First start, insert data and upload it to the remote storage
     env = neon_env_builder.init_start()
@@ -433,9 +431,7 @@ def test_timeline_delete_fail_before_local_delete(neon_env_builder: NeonEnvBuild
     the deletion of the local state.
     """
 
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=RemoteStorageKind.MOCK_S3,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.MOCK_S3)
 
     env = neon_env_builder.init_start()
 
@@ -552,9 +548,7 @@ def test_concurrent_timeline_delete_stuck_on(
     signalling to console that it should retry later.
     """
 
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=RemoteStorageKind.MOCK_S3,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.MOCK_S3)
 
     env = neon_env_builder.init_start()
 
@@ -629,9 +623,7 @@ def test_delete_timeline_client_hangup(neon_env_builder: NeonEnvBuilder):
 
     This tests cancel safety up to the given failpoint.
     """
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=RemoteStorageKind.MOCK_S3,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.MOCK_S3)
 
     env = neon_env_builder.init_start()
 
@@ -698,9 +690,7 @@ def test_timeline_delete_works_for_remote_smoke(
     neon_env_builder: NeonEnvBuilder,
     remote_storage_kind: RemoteStorageKind,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start()
 
@@ -773,7 +763,7 @@ def test_delete_orphaned_objects(
     pg_bin: PgBin,
 ):
     remote_storage_kind = RemoteStorageKind.LOCAL_FS
-    neon_env_builder.enable_remote_storage(remote_storage_kind)
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start(
         initial_tenant_conf={
@@ -784,7 +774,7 @@ def test_delete_orphaned_objects(
         }
     )
 
-    assert isinstance(env.remote_storage, LocalFsStorage)
+    assert isinstance(env.pageserver_remote_storage, LocalFsStorage)
 
     ps_http = env.pageserver.http_client()
 
@@ -795,7 +785,9 @@ def test_delete_orphaned_objects(
         last_flush_lsn_upload(env, endpoint, env.initial_tenant, timeline_id)
 
     # write orphaned file that is missing from the index
-    remote_timeline_path = env.remote_storage.timeline_path(env.initial_tenant, timeline_id)
+    remote_timeline_path = env.pageserver_remote_storage.timeline_path(
+        env.initial_tenant, timeline_id
+    )
     orphans = [remote_timeline_path / f"orphan_{i}" for i in range(3)]
     for orphan in orphans:
         orphan.write_text("I shouldnt be there")
@@ -826,7 +818,7 @@ def test_delete_orphaned_objects(
             f"deleting a file not referenced from index_part.json name={orphan.stem}"
         )
 
-    assert env.remote_storage.index_path(env.initial_tenant, timeline_id).exists()
+    assert env.pageserver_remote_storage.index_path(env.initial_tenant, timeline_id).exists()
 
 
 @pytest.mark.parametrize("remote_storage_kind", available_remote_storages())
@@ -835,9 +827,7 @@ def test_timeline_delete_resumed_on_attach(
     remote_storage_kind: RemoteStorageKind,
     pg_bin: PgBin,
 ):
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
+    neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start(initial_tenant_conf=MANY_SMALL_LAYERS_TENANT_CONFIG)
 
diff --git a/test_runner/regress/test_timeline_size.py b/test_runner/regress/test_timeline_size.py
index d754ce0aa0..ccdfc18de8 100644
--- a/test_runner/regress/test_timeline_size.py
+++ b/test_runner/regress/test_timeline_size.py
@@ -306,7 +306,7 @@ def test_timeline_physical_size_init(
     neon_env_builder: NeonEnvBuilder, remote_storage_kind: Optional[RemoteStorageKind]
 ):
     if remote_storage_kind is not None:
-        neon_env_builder.enable_remote_storage(remote_storage_kind)
+        neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start()
 
@@ -347,7 +347,7 @@ def test_timeline_physical_size_post_checkpoint(
     neon_env_builder: NeonEnvBuilder, remote_storage_kind: Optional[RemoteStorageKind]
 ):
     if remote_storage_kind is not None:
-        neon_env_builder.enable_remote_storage(remote_storage_kind)
+        neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start()
 
@@ -378,7 +378,7 @@ def test_timeline_physical_size_post_compaction(
     neon_env_builder: NeonEnvBuilder, remote_storage_kind: Optional[RemoteStorageKind]
 ):
     if remote_storage_kind is not None:
-        neon_env_builder.enable_remote_storage(remote_storage_kind)
+        neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     # Disable background compaction as we don't want it to happen after `get_physical_size` request
     # and before checking the expected size on disk, which makes the assertion failed
@@ -431,7 +431,7 @@ def test_timeline_physical_size_post_gc(
     neon_env_builder: NeonEnvBuilder, remote_storage_kind: Optional[RemoteStorageKind]
 ):
     if remote_storage_kind is not None:
-        neon_env_builder.enable_remote_storage(remote_storage_kind)
+        neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     # Disable background compaction and GC as we don't want it to happen after `get_physical_size` request
     # and before checking the expected size on disk, which makes the assertion failed
@@ -564,7 +564,7 @@ def test_tenant_physical_size(
     random.seed(100)
 
     if remote_storage_kind is not None:
-        neon_env_builder.enable_remote_storage(remote_storage_kind)
+        neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start()
 
diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py
index 119a597d43..b080caca33 100644
--- a/test_runner/regress/test_wal_acceptor.py
+++ b/test_runner/regress/test_wal_acceptor.py
@@ -37,7 +37,6 @@ from fixtures.pg_version import PgVersion
 from fixtures.port_distributor import PortDistributor
 from fixtures.remote_storage import (
     RemoteStorageKind,
-    RemoteStorageUsers,
     available_remote_storages,
 )
 from fixtures.types import Lsn, TenantId, TimelineId
@@ -267,7 +266,7 @@ def test_restarts(neon_env_builder: NeonEnvBuilder):
 # Test that safekeepers push their info to the broker and learn peer status from it
 def test_broker(neon_env_builder: NeonEnvBuilder):
     neon_env_builder.num_safekeepers = 3
-    neon_env_builder.enable_local_fs_remote_storage()
+    neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS)
     env = neon_env_builder.init_start()
 
     tenant_id = env.initial_tenant
@@ -313,7 +312,7 @@ def test_broker(neon_env_builder: NeonEnvBuilder):
 def test_wal_removal(neon_env_builder: NeonEnvBuilder, auth_enabled: bool):
     neon_env_builder.num_safekeepers = 2
     # to advance remote_consistent_lsn
-    neon_env_builder.enable_local_fs_remote_storage()
+    neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS)
     neon_env_builder.auth_enabled = auth_enabled
     env = neon_env_builder.init_start()
 
@@ -436,12 +435,7 @@ def is_wal_trimmed(sk: Safekeeper, tenant_id: TenantId, timeline_id: TimelineId,
 @pytest.mark.parametrize("remote_storage_kind", available_remote_storages())
 def test_wal_backup(neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind):
     neon_env_builder.num_safekeepers = 3
-
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
-
-    neon_env_builder.remote_storage_users = RemoteStorageUsers.SAFEKEEPER
+    neon_env_builder.enable_safekeeper_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start()
 
@@ -488,11 +482,7 @@ def test_wal_backup(neon_env_builder: NeonEnvBuilder, remote_storage_kind: Remot
 def test_s3_wal_replay(neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind):
     neon_env_builder.num_safekeepers = 3
 
-    neon_env_builder.enable_remote_storage(
-        remote_storage_kind=remote_storage_kind,
-    )
-
-    neon_env_builder.remote_storage_users = RemoteStorageUsers.SAFEKEEPER
+    neon_env_builder.enable_safekeeper_remote_storage(remote_storage_kind)
 
     env = neon_env_builder.init_start()
     tenant_id = env.initial_tenant

From 720d59737acaa8a9b280811b274ca9033006cd1d Mon Sep 17 00:00:00 2001
From: Joonas Koivunen <joonas@neon.tech>
Date: Fri, 8 Sep 2023 16:28:41 +0300
Subject: [PATCH 39/61] rust-1.72.0 changes (#5255)

Prepare to upgrade rust version to latest stable.

- `rustfmt` has learned to format `let irrefutable = $expr else { ...
};` blocks
- There's a new warning about virtual (workspace) crate resolver, picked
the latest resolver as I suspect everyone would expect it to be the
latest; should not matter anyways
- Some new clippies, which seem alright
---
 Cargo.toml                                          | 1 +
 libs/pq_proto/src/lib.rs                            | 2 +-
 libs/remote_storage/src/s3_bucket.rs                | 2 +-
 pageserver/src/config.rs                            | 4 ++--
 pageserver/src/tenant/storage_layer/remote_layer.rs | 2 +-
 s3_scrubber/src/checks.rs                           | 5 ++++-
 6 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 9e2a32ba52..847865cfe8 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,4 +1,5 @@
 [workspace]
+resolver = "2"
 members = [
     "compute_tools",
     "control_plane",
diff --git a/libs/pq_proto/src/lib.rs b/libs/pq_proto/src/lib.rs
index 809fa5fffd..47faad363f 100644
--- a/libs/pq_proto/src/lib.rs
+++ b/libs/pq_proto/src/lib.rs
@@ -959,7 +959,7 @@ mod tests {
         let make_params = |options| StartupMessageParams::new([("options", options)]);
 
         let params = StartupMessageParams::new([]);
-        assert!(matches!(params.options_escaped(), None));
+        assert!(params.options_escaped().is_none());
 
         let params = make_params("");
         assert!(split_options(&params).is_empty());
diff --git a/libs/remote_storage/src/s3_bucket.rs b/libs/remote_storage/src/s3_bucket.rs
index 9d05fa32b3..9262f1e88f 100644
--- a/libs/remote_storage/src/s3_bucket.rs
+++ b/libs/remote_storage/src/s3_bucket.rs
@@ -573,7 +573,7 @@ mod tests {
 
     #[test]
     fn relative_path() {
-        let all_paths = vec!["", "some/path", "some/path/"];
+        let all_paths = ["", "some/path", "some/path/"];
         let all_paths: Vec<RemotePath> = all_paths
             .iter()
             .map(|x| RemotePath::new(Path::new(x)).expect("bad path"))
diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs
index 624503a3a5..89ba3b6310 100644
--- a/pageserver/src/config.rs
+++ b/pageserver/src/config.rs
@@ -73,7 +73,7 @@ pub mod defaults {
     /// Default built-in configuration file.
     ///
     pub const DEFAULT_CONFIG_FILE: &str = formatcp!(
-        r###"
+        r#"
 # Initial configuration file created by 'pageserver --init'
 #listen_pg_addr = '{DEFAULT_PG_LISTEN_ADDR}'
 #listen_http_addr = '{DEFAULT_HTTP_LISTEN_ADDR}'
@@ -118,7 +118,7 @@ pub mod defaults {
 
 [remote_storage]
 
-"###
+"#
     );
 }
 
diff --git a/pageserver/src/tenant/storage_layer/remote_layer.rs b/pageserver/src/tenant/storage_layer/remote_layer.rs
index 3f8d700863..3968c16c31 100644
--- a/pageserver/src/tenant/storage_layer/remote_layer.rs
+++ b/pageserver/src/tenant/storage_layer/remote_layer.rs
@@ -74,7 +74,7 @@ impl Layer for RemoteLayer {
         _reconstruct_state: &mut ValueReconstructState,
         _ctx: &RequestContext,
     ) -> Result<ValueReconstructResult> {
-        bail!("layer {self} needs to be downloaded");
+        Err(anyhow::anyhow!("layer {self} needs to be downloaded"))
     }
 }
 
diff --git a/s3_scrubber/src/checks.rs b/s3_scrubber/src/checks.rs
index e86c381985..914681d707 100644
--- a/s3_scrubber/src/checks.rs
+++ b/s3_scrubber/src/checks.rs
@@ -46,7 +46,10 @@ pub async fn validate_pageserver_active_tenant_and_timelines(
             .push(active_branch);
 
         let Some(active_project) = s3_active_projects.get(&active_project_id) else {
-            error!("Branch {:?} for project {:?} has no such project in the active projects", active_branch_id, active_project_id);
+            error!(
+                "Branch {:?} for project {:?} has no such project in the active projects",
+                active_branch_id, active_project_id
+            );
             continue;
         };
 

From 499d0707d25f653175a51902de75513af3f7c659 Mon Sep 17 00:00:00 2001
From: Konstantin Knizhnik <knizhnik@garret.ru>
Date: Fri, 8 Sep 2023 18:05:08 +0300
Subject: [PATCH 40/61] Perform throttling for concurrent build index which is
 done outside transaction (#5048)

See
https://neondb.slack.com/archives/C03H1K0PGKH/p1692550646191429

## Problem

Build index concurrently is writing WAL outside transaction.
`backpressure_throttling_impl` doesn't perform throttling for read-only
transactions (not assigned XID).
It cause huge write lag which can cause large delay of accessing the
table.

## Summary of changes

Looks at `PROC_IN_SAFE_IC` in process state set during concurrent index
build.

## Checklist before requesting a review

- [ ] I have performed a self-review of my code.
- [ ] If it is a core feature, I have added thorough tests.
- [ ] Do we need to implement analytics? if so did you add the relevant
metrics to the dashboard?
- [ ] If this PR requires public announcement, mark it with
/release-notes label and add several sentences in this section.

## Checklist before merging

- [ ] Do not forget to reformat commit message to not include the above
checklist

---------

Co-authored-by: Konstantin Knizhnik <knizhnik@neon.tech>
Co-authored-by: Heikki Linnakangas <heikki@neon.tech>
---
 pgxn/neon/walproposer.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/pgxn/neon/walproposer.c b/pgxn/neon/walproposer.c
index d9999ef2b1..7ca335945d 100644
--- a/pgxn/neon/walproposer.c
+++ b/pgxn/neon/walproposer.c
@@ -2539,8 +2539,15 @@ backpressure_throttling_impl(void)
 	? PrevProcessInterruptsCallback()
 	: false;
 
-	/* Don't throttle read only transactions and wal sender. */
-	if (am_walsender || !TransactionIdIsValid(GetCurrentTransactionIdIfAny()))
+	/*
+	 * Don't throttle read only transactions or wal sender.
+	 * Do throttle CREATE INDEX CONCURRENTLY, however. It performs some
+	 * stages outside a transaction, even though it writes a lot of WAL. 
+	 * Check PROC_IN_SAFE_IC flag to cover that case.
+	 */
+	if (am_walsender
+		|| (!(MyProc->statusFlags & PROC_IN_SAFE_IC)
+			&& !TransactionIdIsValid(GetCurrentTransactionIdIfAny())))
 		return retry;
 
 	/* Calculate replicas lag */

From 7b6337db583861811c1ee187aa2249095e204da9 Mon Sep 17 00:00:00 2001
From: John Spray <john@neon.tech>
Date: Fri, 8 Sep 2023 16:19:57 +0100
Subject: [PATCH 41/61] tests: enable multiple pageservers in `neon_local` and
 `neon_fixture` (#5231)

## Problem

Currently our testing environment only supports running a single
pageserver at a time. This is insufficient for testing failover and
migrations.
- Dependency of writing tests for #5207

## Summary of changes

- `neon_local` and `neon_fixture` now handle multiple pageservers
- This is a breaking change to the `.neon/config` format: any local
environments will need recreating
- Existing tests continue to work unchanged:
  - The default number of pageservers is 1
- `NeonEnv.pageserver` is now a helper property that retrieves the first
pageserver if there is only one, else throws.
- Pageserver data directories are now at `.neon/pageserver_{n}` where n
is 1,2,3...
- Compatibility tests get some special casing to migrate neon_local
configs: these are not meant to be backward/forward compatible, but they
were treated that way by the test.
---
 control_plane/simple.conf                     |   3 +-
 control_plane/src/attachment_service.rs       |   3 +-
 control_plane/src/bin/neon_local.rs           | 154 ++++++++----
 control_plane/src/endpoint.rs                 |  26 +-
 control_plane/src/local_env.rs                |  38 ++-
 control_plane/src/pageserver.rs               |  61 +++--
 test_runner/fixtures/neon_fixtures.py         | 223 +++++++++++++-----
 test_runner/regress/test_broken_timeline.py   |  14 +-
 test_runner/regress/test_close_fds.py         |   2 +-
 test_runner/regress/test_compatibility.py     |  79 ++++---
 test_runner/regress/test_import.py            |   2 +-
 test_runner/regress/test_large_schema.py      |   2 +-
 test_runner/regress/test_neon_cli.py          |  38 ++-
 test_runner/regress/test_ondemand_download.py |   6 +-
 test_runner/regress/test_pageserver_api.py    |   6 +-
 test_runner/regress/test_read_trace.py        |   2 +-
 test_runner/regress/test_remote_storage.py    |   6 +-
 test_runner/regress/test_tenant_conf.py       |   2 +-
 test_runner/regress/test_tenant_delete.py     |   3 +-
 test_runner/regress/test_tenant_detach.py     |  14 +-
 test_runner/regress/test_tenant_relocation.py |   4 +-
 test_runner/regress/test_tenants.py           |  11 +-
 .../test_tenants_with_remote_storage.py       |   4 +-
 test_runner/regress/test_timeline_delete.py   |  22 +-
 test_runner/regress/test_wal_restore.py       |   2 +-
 .../test_walredo_not_left_behind_on_detach.py |   6 +-
 26 files changed, 497 insertions(+), 236 deletions(-)

diff --git a/control_plane/simple.conf b/control_plane/simple.conf
index 243e13f3d3..0ad90a4618 100644
--- a/control_plane/simple.conf
+++ b/control_plane/simple.conf
@@ -1,6 +1,7 @@
 # Minimal neon environment with one safekeeper. This is equivalent to the built-in
 # defaults that you get with no --config
-[pageserver]
+[[pageservers]]
+id=1
 listen_pg_addr = '127.0.0.1:64000'
 listen_http_addr = '127.0.0.1:9898'
 pg_auth_type = 'Trust'
diff --git a/control_plane/src/attachment_service.rs b/control_plane/src/attachment_service.rs
index 2bd4260aa8..f0e649cfa8 100644
--- a/control_plane/src/attachment_service.rs
+++ b/control_plane/src/attachment_service.rs
@@ -32,7 +32,7 @@ impl AttachmentService {
 
         // Makes no sense to construct this if pageservers aren't going to use it: assume
         // pageservers have control plane API set
-        let listen_url = env.pageserver.control_plane_api.clone().unwrap();
+        let listen_url = env.control_plane_api.clone().unwrap();
 
         let listen = format!(
             "{}:{}",
@@ -80,7 +80,6 @@ impl AttachmentService {
 
         let url = self
             .env
-            .pageserver
             .control_plane_api
             .clone()
             .unwrap()
diff --git a/control_plane/src/bin/neon_local.rs b/control_plane/src/bin/neon_local.rs
index 6b49b92cfa..4cdb91bfd2 100644
--- a/control_plane/src/bin/neon_local.rs
+++ b/control_plane/src/bin/neon_local.rs
@@ -50,16 +50,17 @@ fn default_conf() -> String {
     format!(
         r#"
 # Default built-in configuration, defined in main.rs
+control_plane_api = '{DEFAULT_PAGESERVER_CONTROL_PLANE_API}'
+
 [broker]
 listen_addr = '{DEFAULT_BROKER_ADDR}'
 
-[pageserver]
+[[pageservers]]
 id = {DEFAULT_PAGESERVER_ID}
 listen_pg_addr = '{DEFAULT_PAGESERVER_PG_ADDR}'
 listen_http_addr = '{DEFAULT_PAGESERVER_HTTP_ADDR}'
 pg_auth_type = '{trust_auth}'
 http_auth_type = '{trust_auth}'
-control_plane_api = '{DEFAULT_PAGESERVER_CONTROL_PLANE_API}'
 
 [[safekeepers]]
 id = {DEFAULT_SAFEKEEPER_ID}
@@ -258,7 +259,7 @@ fn get_timeline_infos(
     env: &local_env::LocalEnv,
     tenant_id: &TenantId,
 ) -> Result<HashMap<TimelineId, TimelineInfo>> {
-    Ok(PageServerNode::from_env(env)
+    Ok(get_default_pageserver(env)
         .timeline_list(tenant_id)?
         .into_iter()
         .map(|timeline_info| (timeline_info.timeline_id, timeline_info))
@@ -319,17 +320,30 @@ fn handle_init(init_match: &ArgMatches) -> anyhow::Result<LocalEnv> {
         .context("Failed to initialize neon repository")?;
 
     // Initialize pageserver, create initial tenant and timeline.
-    let pageserver = PageServerNode::from_env(&env);
-    pageserver
-        .initialize(&pageserver_config_overrides(init_match))
-        .unwrap_or_else(|e| {
-            eprintln!("pageserver init failed: {e:?}");
-            exit(1);
-        });
+    for ps_conf in &env.pageservers {
+        PageServerNode::from_env(&env, ps_conf)
+            .initialize(&pageserver_config_overrides(init_match))
+            .unwrap_or_else(|e| {
+                eprintln!("pageserver init failed: {e:?}");
+                exit(1);
+            });
+    }
 
     Ok(env)
 }
 
+/// The default pageserver is the one where CLI tenant/timeline operations are sent by default.
+/// For typical interactive use, one would just run with a single pageserver.  Scenarios with
+/// tenant/timeline placement across multiple pageservers are managed by python test code rather
+/// than this CLI.
+fn get_default_pageserver(env: &local_env::LocalEnv) -> PageServerNode {
+    let ps_conf = env
+        .pageservers
+        .first()
+        .expect("Config is validated to contain at least one pageserver");
+    PageServerNode::from_env(env, ps_conf)
+}
+
 fn pageserver_config_overrides(init_match: &ArgMatches) -> Vec<&str> {
     init_match
         .get_many::<String>("pageserver-config-override")
@@ -340,7 +354,7 @@ fn pageserver_config_overrides(init_match: &ArgMatches) -> Vec<&str> {
 }
 
 fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> anyhow::Result<()> {
-    let pageserver = PageServerNode::from_env(env);
+    let pageserver = get_default_pageserver(env);
     match tenant_match.subcommand() {
         Some(("list", _)) => {
             for t in pageserver.tenant_list()? {
@@ -356,11 +370,11 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> an
             // If tenant ID was not specified, generate one
             let tenant_id = parse_tenant_id(create_match)?.unwrap_or_else(TenantId::generate);
 
-            let generation = if env.pageserver.control_plane_api.is_some() {
+            let generation = if env.control_plane_api.is_some() {
                 // We must register the tenant with the attachment service, so
                 // that when the pageserver restarts, it will be re-attached.
                 let attachment_service = AttachmentService::from_env(env);
-                attachment_service.attach_hook(tenant_id, env.pageserver.id)?
+                attachment_service.attach_hook(tenant_id, pageserver.conf.id)?
             } else {
                 None
             };
@@ -425,7 +439,7 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> an
 }
 
 fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) -> Result<()> {
-    let pageserver = PageServerNode::from_env(env);
+    let pageserver = get_default_pageserver(env);
 
     match timeline_match.subcommand() {
         Some(("list", list_match)) => {
@@ -502,6 +516,7 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) -
                 None,
                 pg_version,
                 ComputeMode::Primary,
+                DEFAULT_PAGESERVER_ID,
             )?;
             println!("Done");
         }
@@ -555,7 +570,6 @@ fn handle_endpoint(ep_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<(
         Some(ep_subcommand_data) => ep_subcommand_data,
         None => bail!("no endpoint subcommand provided"),
     };
-
     let mut cplane = ComputeControlPlane::load(env.clone())?;
 
     // All subcommands take an optional --tenant-id option
@@ -652,6 +666,13 @@ fn handle_endpoint(ep_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<(
                 .copied()
                 .unwrap_or(false);
 
+            let pageserver_id =
+                if let Some(id_str) = sub_args.get_one::<String>("endpoint-pageserver-id") {
+                    NodeId(id_str.parse().context("while parsing pageserver id")?)
+                } else {
+                    DEFAULT_PAGESERVER_ID
+                };
+
             let mode = match (lsn, hot_standby) {
                 (Some(lsn), false) => ComputeMode::Static(lsn),
                 (None, true) => ComputeMode::Replica,
@@ -667,6 +688,7 @@ fn handle_endpoint(ep_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<(
                 http_port,
                 pg_version,
                 mode,
+                pageserver_id,
             )?;
         }
         "start" => {
@@ -676,6 +698,13 @@ fn handle_endpoint(ep_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<(
                 .get_one::<String>("endpoint_id")
                 .ok_or_else(|| anyhow!("No endpoint ID was provided to start"))?;
 
+            let pageserver_id =
+                if let Some(id_str) = sub_args.get_one::<String>("endpoint-pageserver-id") {
+                    NodeId(id_str.parse().context("while parsing pageserver id")?)
+                } else {
+                    DEFAULT_PAGESERVER_ID
+                };
+
             let remote_ext_config = sub_args.get_one::<String>("remote-ext-config");
 
             // If --safekeepers argument is given, use only the listed safekeeper nodes.
@@ -695,7 +724,8 @@ fn handle_endpoint(ep_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<(
 
             let endpoint = cplane.endpoints.get(endpoint_id.as_str());
 
-            let auth_token = if matches!(env.pageserver.pg_auth_type, AuthType::NeonJWT) {
+            let ps_conf = env.get_pageserver_conf(pageserver_id)?;
+            let auth_token = if matches!(ps_conf.pg_auth_type, AuthType::NeonJWT) {
                 let claims = Claims::new(Some(tenant_id), Scope::Tenant);
 
                 Some(env.generate_auth_token(&claims)?)
@@ -762,6 +792,7 @@ fn handle_endpoint(ep_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<(
                     http_port,
                     pg_version,
                     mode,
+                    pageserver_id,
                 )?;
                 ep.start(&auth_token, safekeepers, remote_ext_config)?;
             }
@@ -786,48 +817,64 @@ fn handle_endpoint(ep_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<(
 }
 
 fn handle_pageserver(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> {
-    let pageserver = PageServerNode::from_env(env);
+    fn get_pageserver(env: &local_env::LocalEnv, args: &ArgMatches) -> Result<PageServerNode> {
+        let node_id = if let Some(id_str) = args.get_one::<String>("pageserver-id") {
+            NodeId(id_str.parse().context("while parsing pageserver id")?)
+        } else {
+            DEFAULT_PAGESERVER_ID
+        };
+
+        Ok(PageServerNode::from_env(
+            env,
+            env.get_pageserver_conf(node_id)?,
+        ))
+    }
 
     match sub_match.subcommand() {
-        Some(("start", start_match)) => {
-            if let Err(e) = pageserver.start(&pageserver_config_overrides(start_match)) {
+        Some(("start", subcommand_args)) => {
+            if let Err(e) = get_pageserver(env, subcommand_args)?
+                .start(&pageserver_config_overrides(subcommand_args))
+            {
                 eprintln!("pageserver start failed: {e}");
                 exit(1);
             }
         }
 
-        Some(("stop", stop_match)) => {
-            let immediate = stop_match
+        Some(("stop", subcommand_args)) => {
+            let immediate = subcommand_args
                 .get_one::<String>("stop-mode")
                 .map(|s| s.as_str())
                 == Some("immediate");
 
-            if let Err(e) = pageserver.stop(immediate) {
+            if let Err(e) = get_pageserver(env, subcommand_args)?.stop(immediate) {
                 eprintln!("pageserver stop failed: {}", e);
                 exit(1);
             }
         }
 
-        Some(("restart", restart_match)) => {
+        Some(("restart", subcommand_args)) => {
+            let pageserver = get_pageserver(env, subcommand_args)?;
             //TODO what shutdown strategy should we use here?
             if let Err(e) = pageserver.stop(false) {
                 eprintln!("pageserver stop failed: {}", e);
                 exit(1);
             }
 
-            if let Err(e) = pageserver.start(&pageserver_config_overrides(restart_match)) {
+            if let Err(e) = pageserver.start(&pageserver_config_overrides(subcommand_args)) {
                 eprintln!("pageserver start failed: {e}");
                 exit(1);
             }
         }
 
-        Some(("status", _)) => match PageServerNode::from_env(env).check_status() {
-            Ok(_) => println!("Page server is up and running"),
-            Err(err) => {
-                eprintln!("Page server is not available: {}", err);
-                exit(1);
+        Some(("status", subcommand_args)) => {
+            match get_pageserver(env, subcommand_args)?.check_status() {
+                Ok(_) => println!("Page server is up and running"),
+                Err(err) => {
+                    eprintln!("Page server is not available: {}", err);
+                    exit(1);
+                }
             }
-        },
+        }
 
         Some((sub_name, _)) => bail!("Unexpected pageserver subcommand '{}'", sub_name),
         None => bail!("no pageserver subcommand provided"),
@@ -943,7 +990,7 @@ fn handle_start_all(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> anyhow
     broker::start_broker_process(env)?;
 
     // Only start the attachment service if the pageserver is configured to need it
-    if env.pageserver.control_plane_api.is_some() {
+    if env.control_plane_api.is_some() {
         let attachment_service = AttachmentService::from_env(env);
         if let Err(e) = attachment_service.start() {
             eprintln!("attachment_service start failed: {:#}", e);
@@ -952,11 +999,13 @@ fn handle_start_all(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> anyhow
         }
     }
 
-    let pageserver = PageServerNode::from_env(env);
-    if let Err(e) = pageserver.start(&pageserver_config_overrides(sub_match)) {
-        eprintln!("pageserver {} start failed: {:#}", env.pageserver.id, e);
-        try_stop_all(env, true);
-        exit(1);
+    for ps_conf in &env.pageservers {
+        let pageserver = PageServerNode::from_env(env, ps_conf);
+        if let Err(e) = pageserver.start(&pageserver_config_overrides(sub_match)) {
+            eprintln!("pageserver {} start failed: {:#}", ps_conf.id, e);
+            try_stop_all(env, true);
+            exit(1);
+        }
     }
 
     for node in env.safekeepers.iter() {
@@ -980,8 +1029,6 @@ fn handle_stop_all(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<
 }
 
 fn try_stop_all(env: &local_env::LocalEnv, immediate: bool) {
-    let pageserver = PageServerNode::from_env(env);
-
     // Stop all endpoints
     match ComputeControlPlane::load(env.clone()) {
         Ok(cplane) => {
@@ -996,8 +1043,11 @@ fn try_stop_all(env: &local_env::LocalEnv, immediate: bool) {
         }
     }
 
-    if let Err(e) = pageserver.stop(immediate) {
-        eprintln!("pageserver {} stop failed: {:#}", env.pageserver.id, e);
+    for ps_conf in &env.pageservers {
+        let pageserver = PageServerNode::from_env(env, ps_conf);
+        if let Err(e) = pageserver.stop(immediate) {
+            eprintln!("pageserver {} stop failed: {:#}", ps_conf.id, e);
+        }
     }
 
     for node in env.safekeepers.iter() {
@@ -1011,7 +1061,7 @@ fn try_stop_all(env: &local_env::LocalEnv, immediate: bool) {
         eprintln!("neon broker stop failed: {e:#}");
     }
 
-    if env.pageserver.control_plane_api.is_some() {
+    if env.control_plane_api.is_some() {
         let attachment_service = AttachmentService::from_env(env);
         if let Err(e) = attachment_service.stop(immediate) {
             eprintln!("attachment service stop failed: {e:#}");
@@ -1031,6 +1081,16 @@ fn cli() -> Command {
 
     let safekeeper_id_arg = Arg::new("id").help("safekeeper id").required(false);
 
+    // --id, when using a pageserver command
+    let pageserver_id_arg = Arg::new("pageserver-id")
+        .long("id")
+        .help("pageserver id")
+        .required(false);
+    // --pageserver-id when using a non-pageserver command
+    let endpoint_pageserver_id_arg = Arg::new("endpoint-pageserver-id")
+        .long("pageserver-id")
+        .required(false);
+
     let safekeeper_extra_opt_arg = Arg::new("safekeeper-extra-opt")
         .short('e')
         .long("safekeeper-extra-opt")
@@ -1195,10 +1255,16 @@ fn cli() -> Command {
                 .arg_required_else_help(true)
                 .about("Manage pageserver")
                 .subcommand(Command::new("status"))
-                .subcommand(Command::new("start").about("Start local pageserver").arg(pageserver_config_args.clone()))
+                .arg(pageserver_id_arg.clone())
+                .subcommand(Command::new("start").about("Start local pageserver")
+                .arg(pageserver_id_arg.clone())
+                .arg(pageserver_config_args.clone()))
                 .subcommand(Command::new("stop").about("Stop local pageserver")
+                .arg(pageserver_id_arg.clone())
                             .arg(stop_mode_arg.clone()))
-                .subcommand(Command::new("restart").about("Restart local pageserver").arg(pageserver_config_args.clone()))
+                .subcommand(Command::new("restart").about("Restart local pageserver")
+                .arg(pageserver_id_arg.clone())
+                .arg(pageserver_config_args.clone()))
         )
         .subcommand(
             Command::new("attachment_service")
@@ -1242,6 +1308,7 @@ fn cli() -> Command {
                     .arg(lsn_arg.clone())
                     .arg(pg_port_arg.clone())
                     .arg(http_port_arg.clone())
+                    .arg(endpoint_pageserver_id_arg.clone())
                     .arg(
                         Arg::new("config-only")
                             .help("Don't do basebackup, create endpoint directory with only config files")
@@ -1259,6 +1326,7 @@ fn cli() -> Command {
                     .arg(lsn_arg)
                     .arg(pg_port_arg)
                     .arg(http_port_arg)
+                    .arg(endpoint_pageserver_id_arg.clone())
                     .arg(pg_version_arg)
                     .arg(hot_standby_arg)
                     .arg(safekeepers_arg)
diff --git a/control_plane/src/endpoint.rs b/control_plane/src/endpoint.rs
index 4ed03c8771..cba364c049 100644
--- a/control_plane/src/endpoint.rs
+++ b/control_plane/src/endpoint.rs
@@ -70,6 +70,7 @@ pub struct EndpointConf {
     http_port: u16,
     pg_version: u32,
     skip_pg_catalog_updates: bool,
+    pageserver_id: NodeId,
 }
 
 //
@@ -82,19 +83,16 @@ pub struct ComputeControlPlane {
     pub endpoints: BTreeMap<String, Arc<Endpoint>>,
 
     env: LocalEnv,
-    pageserver: Arc<PageServerNode>,
 }
 
 impl ComputeControlPlane {
     // Load current endpoints from the endpoints/ subdirectories
     pub fn load(env: LocalEnv) -> Result<ComputeControlPlane> {
-        let pageserver = Arc::new(PageServerNode::from_env(&env));
-
         let mut endpoints = BTreeMap::default();
         for endpoint_dir in std::fs::read_dir(env.endpoints_path())
             .with_context(|| format!("failed to list {}", env.endpoints_path().display()))?
         {
-            let ep = Endpoint::from_dir_entry(endpoint_dir?, &env, &pageserver)?;
+            let ep = Endpoint::from_dir_entry(endpoint_dir?, &env)?;
             endpoints.insert(ep.endpoint_id.clone(), Arc::new(ep));
         }
 
@@ -102,7 +100,6 @@ impl ComputeControlPlane {
             base_port: 55431,
             endpoints,
             env,
-            pageserver,
         })
     }
 
@@ -125,15 +122,18 @@ impl ComputeControlPlane {
         http_port: Option<u16>,
         pg_version: u32,
         mode: ComputeMode,
+        pageserver_id: NodeId,
     ) -> Result<Arc<Endpoint>> {
         let pg_port = pg_port.unwrap_or_else(|| self.get_port());
         let http_port = http_port.unwrap_or_else(|| self.get_port() + 1);
+        let pageserver =
+            PageServerNode::from_env(&self.env, self.env.get_pageserver_conf(pageserver_id)?);
         let ep = Arc::new(Endpoint {
             endpoint_id: endpoint_id.to_owned(),
             pg_address: SocketAddr::new("127.0.0.1".parse().unwrap(), pg_port),
             http_address: SocketAddr::new("127.0.0.1".parse().unwrap(), http_port),
             env: self.env.clone(),
-            pageserver: Arc::clone(&self.pageserver),
+            pageserver,
             timeline_id,
             mode,
             tenant_id,
@@ -159,6 +159,7 @@ impl ComputeControlPlane {
                 pg_port,
                 pg_version,
                 skip_pg_catalog_updates: true,
+                pageserver_id,
             })?,
         )?;
         std::fs::write(
@@ -193,18 +194,14 @@ pub struct Endpoint {
     // These are not part of the endpoint as such, but the environment
     // the endpoint runs in.
     pub env: LocalEnv,
-    pageserver: Arc<PageServerNode>,
+    pageserver: PageServerNode,
 
     // Optimizations
     skip_pg_catalog_updates: bool,
 }
 
 impl Endpoint {
-    fn from_dir_entry(
-        entry: std::fs::DirEntry,
-        env: &LocalEnv,
-        pageserver: &Arc<PageServerNode>,
-    ) -> Result<Endpoint> {
+    fn from_dir_entry(entry: std::fs::DirEntry, env: &LocalEnv) -> Result<Endpoint> {
         if !entry.file_type()?.is_dir() {
             anyhow::bail!(
                 "Endpoint::from_dir_entry failed: '{}' is not a directory",
@@ -220,12 +217,15 @@ impl Endpoint {
         let conf: EndpointConf =
             serde_json::from_slice(&std::fs::read(entry.path().join("endpoint.json"))?)?;
 
+        let pageserver =
+            PageServerNode::from_env(env, env.get_pageserver_conf(conf.pageserver_id)?);
+
         Ok(Endpoint {
             pg_address: SocketAddr::new("127.0.0.1".parse().unwrap(), conf.pg_port),
             http_address: SocketAddr::new("127.0.0.1".parse().unwrap(), conf.http_port),
             endpoint_id,
             env: env.clone(),
-            pageserver: Arc::clone(pageserver),
+            pageserver,
             timeline_id: conf.timeline_id,
             mode: conf.mode,
             tenant_id: conf.tenant_id,
diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs
index 0215ab1bb5..b4d09b01ab 100644
--- a/control_plane/src/local_env.rs
+++ b/control_plane/src/local_env.rs
@@ -68,11 +68,17 @@ pub struct LocalEnv {
 
     pub broker: NeonBroker,
 
-    pub pageserver: PageServerConf,
+    /// This Vec must always contain at least one pageserver
+    pub pageservers: Vec<PageServerConf>,
 
     #[serde(default)]
     pub safekeepers: Vec<SafekeeperConf>,
 
+    // Control plane location: if None, we will not run attachment_service.  If set, this will
+    // be propagated into each pageserver's configuration.
+    #[serde(default)]
+    pub control_plane_api: Option<Url>,
+
     /// Keep human-readable aliases in memory (and persist them to config), to hide ZId hex strings from the user.
     #[serde(default)]
     // A `HashMap<String, HashMap<TenantId, TimelineId>>` would be more appropriate here,
@@ -118,9 +124,6 @@ pub struct PageServerConf {
     // auth type used for the PG and HTTP ports
     pub pg_auth_type: AuthType,
     pub http_auth_type: AuthType,
-
-    // Control plane location
-    pub control_plane_api: Option<Url>,
 }
 
 impl Default for PageServerConf {
@@ -131,7 +134,6 @@ impl Default for PageServerConf {
             listen_http_addr: String::new(),
             pg_auth_type: AuthType::Trust,
             http_auth_type: AuthType::Trust,
-            control_plane_api: None,
         }
     }
 }
@@ -222,15 +224,23 @@ impl LocalEnv {
         self.base_data_dir.join("endpoints")
     }
 
-    // TODO: move pageserver files into ./pageserver
-    pub fn pageserver_data_dir(&self) -> PathBuf {
-        self.base_data_dir.clone()
+    pub fn pageserver_data_dir(&self, pageserver_id: NodeId) -> PathBuf {
+        self.base_data_dir
+            .join(format!("pageserver_{pageserver_id}"))
     }
 
     pub fn safekeeper_data_dir(&self, data_dir_name: &str) -> PathBuf {
         self.base_data_dir.join("safekeepers").join(data_dir_name)
     }
 
+    pub fn get_pageserver_conf(&self, id: NodeId) -> anyhow::Result<&PageServerConf> {
+        if let Some(conf) = self.pageservers.iter().find(|node| node.id == id) {
+            Ok(conf)
+        } else {
+            bail!("could not find pageserver {id}")
+        }
+    }
+
     pub fn register_branch_mapping(
         &mut self,
         branch_name: String,
@@ -307,6 +317,10 @@ impl LocalEnv {
             env.neon_distrib_dir = env::current_exe()?.parent().unwrap().to_owned();
         }
 
+        if env.pageservers.is_empty() {
+            anyhow::bail!("Configuration must contain at least one pageserver");
+        }
+
         env.base_data_dir = base_path();
 
         Ok(env)
@@ -339,7 +353,7 @@ impl LocalEnv {
         // We read that in, in `create_config`, and fill any missing defaults. Then it's saved
         // to .neon/config. TODO: We lose any formatting and comments along the way, which is
         // a bit sad.
-        let mut conf_content = r#"# This file describes a locale deployment of the page server
+        let mut conf_content = r#"# This file describes a local deployment of the page server
 # and safekeeeper node. It is read by the 'neon_local' command-line
 # utility.
 "#
@@ -469,9 +483,9 @@ impl LocalEnv {
     }
 
     fn auth_keys_needed(&self) -> bool {
-        self.pageserver.pg_auth_type == AuthType::NeonJWT
-            || self.pageserver.http_auth_type == AuthType::NeonJWT
-            || self.safekeepers.iter().any(|sk| sk.auth_enabled)
+        self.pageservers.iter().any(|ps| {
+            ps.pg_auth_type == AuthType::NeonJWT || ps.http_auth_type == AuthType::NeonJWT
+        }) || self.safekeepers.iter().any(|sk| sk.auth_enabled)
     }
 }
 
diff --git a/control_plane/src/pageserver.rs b/control_plane/src/pageserver.rs
index eecc2479ff..a6b675fdb5 100644
--- a/control_plane/src/pageserver.rs
+++ b/control_plane/src/pageserver.rs
@@ -27,6 +27,7 @@ use utils::{
     lsn::Lsn,
 };
 
+use crate::local_env::PageServerConf;
 use crate::{background_process, local_env::LocalEnv};
 
 #[derive(Error, Debug)]
@@ -76,43 +77,40 @@ impl ResponseErrorMessageExt for Response {
 #[derive(Debug)]
 pub struct PageServerNode {
     pub pg_connection_config: PgConnectionConfig,
+    pub conf: PageServerConf,
     pub env: LocalEnv,
     pub http_client: Client,
     pub http_base_url: String,
 }
 
 impl PageServerNode {
-    pub fn from_env(env: &LocalEnv) -> PageServerNode {
-        let (host, port) = parse_host_port(&env.pageserver.listen_pg_addr)
-            .expect("Unable to parse listen_pg_addr");
+    pub fn from_env(env: &LocalEnv, conf: &PageServerConf) -> PageServerNode {
+        let (host, port) =
+            parse_host_port(&conf.listen_pg_addr).expect("Unable to parse listen_pg_addr");
         let port = port.unwrap_or(5432);
         Self {
             pg_connection_config: PgConnectionConfig::new_host_port(host, port),
+            conf: conf.clone(),
             env: env.clone(),
             http_client: Client::new(),
-            http_base_url: format!("http://{}/v1", env.pageserver.listen_http_addr),
+            http_base_url: format!("http://{}/v1", conf.listen_http_addr),
         }
     }
 
     // pageserver conf overrides defined by neon_local configuration.
     fn neon_local_overrides(&self) -> Vec<String> {
-        let id = format!("id={}", self.env.pageserver.id);
+        let id = format!("id={}", self.conf.id);
         // FIXME: the paths should be shell-escaped to handle paths with spaces, quotas etc.
         let pg_distrib_dir_param = format!(
             "pg_distrib_dir='{}'",
             self.env.pg_distrib_dir_raw().display()
         );
 
-        let http_auth_type_param =
-            format!("http_auth_type='{}'", self.env.pageserver.http_auth_type);
-        let listen_http_addr_param = format!(
-            "listen_http_addr='{}'",
-            self.env.pageserver.listen_http_addr
-        );
+        let http_auth_type_param = format!("http_auth_type='{}'", self.conf.http_auth_type);
+        let listen_http_addr_param = format!("listen_http_addr='{}'", self.conf.listen_http_addr);
 
-        let pg_auth_type_param = format!("pg_auth_type='{}'", self.env.pageserver.pg_auth_type);
-        let listen_pg_addr_param =
-            format!("listen_pg_addr='{}'", self.env.pageserver.listen_pg_addr);
+        let pg_auth_type_param = format!("pg_auth_type='{}'", self.conf.pg_auth_type);
+        let listen_pg_addr_param = format!("listen_pg_addr='{}'", self.conf.listen_pg_addr);
 
         let broker_endpoint_param = format!("broker_endpoint='{}'", self.env.broker.client_url());
 
@@ -126,17 +124,18 @@ impl PageServerNode {
             broker_endpoint_param,
         ];
 
-        if let Some(control_plane_api) = &self.env.pageserver.control_plane_api {
+        if let Some(control_plane_api) = &self.env.control_plane_api {
             overrides.push(format!(
                 "control_plane_api='{}'",
                 control_plane_api.as_str()
             ));
         }
 
-        if self.env.pageserver.http_auth_type != AuthType::Trust
-            || self.env.pageserver.pg_auth_type != AuthType::Trust
+        if self.conf.http_auth_type != AuthType::Trust || self.conf.pg_auth_type != AuthType::Trust
         {
-            overrides.push("auth_validation_public_key_path='auth_public_key.pem'".to_owned());
+            // Keys are generated in the toplevel repo dir, pageservers' workdirs
+            // are one level below that, so refer to keys with ../
+            overrides.push("auth_validation_public_key_path='../auth_public_key.pem'".to_owned());
         }
         overrides
     }
@@ -144,16 +143,12 @@ impl PageServerNode {
     /// Initializes a pageserver node by creating its config with the overrides provided.
     pub fn initialize(&self, config_overrides: &[&str]) -> anyhow::Result<()> {
         // First, run `pageserver --init` and wait for it to write a config into FS and exit.
-        self.pageserver_init(config_overrides).with_context(|| {
-            format!(
-                "Failed to run init for pageserver node {}",
-                self.env.pageserver.id,
-            )
-        })
+        self.pageserver_init(config_overrides)
+            .with_context(|| format!("Failed to run init for pageserver node {}", self.conf.id,))
     }
 
     pub fn repo_path(&self) -> PathBuf {
-        self.env.pageserver_data_dir()
+        self.env.pageserver_data_dir(self.conf.id)
     }
 
     /// The pid file is created by the pageserver process, with its pid stored inside.
@@ -169,7 +164,7 @@ impl PageServerNode {
 
     fn pageserver_init(&self, config_overrides: &[&str]) -> anyhow::Result<()> {
         let datadir = self.repo_path();
-        let node_id = self.env.pageserver.id;
+        let node_id = self.conf.id;
         println!(
             "Initializing pageserver node {} at '{}' in {:?}",
             node_id,
@@ -178,6 +173,10 @@ impl PageServerNode {
         );
         io::stdout().flush()?;
 
+        if !datadir.exists() {
+            std::fs::create_dir(&datadir)?;
+        }
+
         let datadir_path_str = datadir.to_str().with_context(|| {
             format!("Cannot start pageserver node {node_id} in path that has no string representation: {datadir:?}")
         })?;
@@ -208,7 +207,7 @@ impl PageServerNode {
         let datadir = self.repo_path();
         print!(
             "Starting pageserver node {} at '{}' in {:?}",
-            self.env.pageserver.id,
+            self.conf.id,
             self.pg_connection_config.raw_address(),
             datadir
         );
@@ -217,7 +216,7 @@ impl PageServerNode {
         let datadir_path_str = datadir.to_str().with_context(|| {
             format!(
                 "Cannot start pageserver node {} in path that has no string representation: {:?}",
-                self.env.pageserver.id, datadir,
+                self.conf.id, datadir,
             )
         })?;
         let mut args = self.pageserver_basic_args(config_overrides, datadir_path_str);
@@ -261,7 +260,7 @@ impl PageServerNode {
         // FIXME: why is this tied to pageserver's auth type? Whether or not the safekeeper
         // needs a token, and how to generate that token, seems independent to whether
         // the pageserver requires a token in incoming requests.
-        Ok(if self.env.pageserver.http_auth_type != AuthType::Trust {
+        Ok(if self.conf.http_auth_type != AuthType::Trust {
             // Generate a token to connect from the pageserver to a safekeeper
             let token = self
                 .env
@@ -286,7 +285,7 @@ impl PageServerNode {
 
     pub fn page_server_psql_client(&self) -> anyhow::Result<postgres::Client> {
         let mut config = self.pg_connection_config.clone();
-        if self.env.pageserver.pg_auth_type == AuthType::NeonJWT {
+        if self.conf.pg_auth_type == AuthType::NeonJWT {
             let token = self
                 .env
                 .generate_auth_token(&Claims::new(None, Scope::PageServerApi))?;
@@ -297,7 +296,7 @@ impl PageServerNode {
 
     fn http_request<U: IntoUrl>(&self, method: Method, url: U) -> anyhow::Result<RequestBuilder> {
         let mut builder = self.http_client.request(method, url);
-        if self.env.pageserver.http_auth_type == AuthType::NeonJWT {
+        if self.conf.http_auth_type == AuthType::NeonJWT {
             let token = self
                 .env
                 .generate_auth_token(&Claims::new(None, Scope::PageServerApi))?;
diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py
index 37524e7f5d..272c32fdcd 100644
--- a/test_runner/fixtures/neon_fixtures.py
+++ b/test_runner/fixtures/neon_fixtures.py
@@ -416,6 +416,7 @@ class NeonEnvBuilder:
         pageserver_remote_storage: Optional[RemoteStorage] = None,
         pageserver_config_override: Optional[str] = None,
         num_safekeepers: int = 1,
+        num_pageservers: int = 1,
         # Use non-standard SK ids to check for various parsing bugs
         safekeepers_id_start: int = 0,
         # fsync is disabled by default to make the tests go faster
@@ -443,6 +444,7 @@ class NeonEnvBuilder:
         self.mock_s3_server: MockS3Server = mock_s3_server
         self.pageserver_config_override = pageserver_config_override
         self.num_safekeepers = num_safekeepers
+        self.num_pageservers = num_pageservers
         self.safekeepers_id_start = safekeepers_id_start
         self.safekeepers_enable_fsync = safekeepers_enable_fsync
         self.auth_enabled = auth_enabled
@@ -613,7 +615,9 @@ class NeonEnvBuilder:
             self.env.endpoints.stop_all()
             for sk in self.env.safekeepers:
                 sk.stop(immediate=True)
-            self.env.pageserver.stop(immediate=True)
+
+            for pageserver in self.env.pageservers:
+                pageserver.stop(immediate=True)
 
             if self.env.attachment_service is not None:
                 self.env.attachment_service.stop(immediate=True)
@@ -644,7 +648,8 @@ class NeonEnvBuilder:
             if cleanup_error is not None:
                 raise cleanup_error
 
-            self.env.pageserver.assert_no_errors()
+            for pageserver in self.env.pageservers:
+                pageserver.assert_no_errors()
 
 
 class NeonEnv:
@@ -664,8 +669,7 @@ class NeonEnv:
 
     postgres - A factory object for creating postgres compute nodes.
 
-    pageserver - An object that contains functions for manipulating and
-        connecting to the pageserver
+    pageservers - An array containing objects representing the pageservers
 
     safekeepers - An array containing objects representing the safekeepers
 
@@ -680,7 +684,7 @@ class NeonEnv:
         the tenant id
     """
 
-    PAGESERVER_ID = 1
+    BASE_PAGESERVER_ID = 1
 
     def __init__(self, config: NeonEnvBuilder):
         self.repo_dir = config.repo_dir
@@ -690,14 +694,20 @@ class NeonEnv:
         self.neon_cli = NeonCli(env=self)
         self.endpoints = EndpointFactory(self)
         self.safekeepers: List[Safekeeper] = []
+        self.pageservers: List[NeonPageserver] = []
         self.broker = config.broker
         self.pageserver_remote_storage = config.pageserver_remote_storage
         self.ext_remote_storage = config.ext_remote_storage
         self.safekeepers_remote_storage = config.sk_remote_storage
         self.pg_version = config.pg_version
+        # Binary path for pageserver, safekeeper, etc
         self.neon_binpath = config.neon_binpath
+        # Binary path for neon_local test-specific binaries: may be overridden
+        # after construction for compat testing
+        self.neon_local_binpath = config.neon_binpath
         self.pg_distrib_dir = config.pg_distrib_dir
         self.endpoint_counter = 0
+        self.pageserver_config_override = config.pageserver_config_override
 
         # generate initial tenant ID here instead of letting 'neon init' generate it,
         # so that we don't need to dig it out of the config file afterwards.
@@ -719,6 +729,13 @@ class NeonEnv:
         """
         )
 
+        if self.control_plane_api is not None:
+            toml += textwrap.dedent(
+                f"""
+                control_plane_api = '{self.control_plane_api}'
+            """
+            )
+
         toml += textwrap.dedent(
             f"""
             [broker]
@@ -727,36 +744,36 @@ class NeonEnv:
         )
 
         # Create config for pageserver
-        pageserver_port = PageserverPort(
-            pg=self.port_distributor.get_port(),
-            http=self.port_distributor.get_port(),
-        )
         http_auth_type = "NeonJWT" if config.auth_enabled else "Trust"
         pg_auth_type = "NeonJWT" if config.auth_enabled else "Trust"
+        for ps_id in range(
+            self.BASE_PAGESERVER_ID, self.BASE_PAGESERVER_ID + config.num_pageservers
+        ):
+            pageserver_port = PageserverPort(
+                pg=self.port_distributor.get_port(),
+                http=self.port_distributor.get_port(),
+            )
 
-        toml += textwrap.dedent(
-            f"""
-            [pageserver]
-            id={self.PAGESERVER_ID}
-            listen_pg_addr = 'localhost:{pageserver_port.pg}'
-            listen_http_addr = 'localhost:{pageserver_port.http}'
-            pg_auth_type = '{pg_auth_type}'
-            http_auth_type = '{http_auth_type}'
-        """
-        )
-
-        if self.control_plane_api is not None:
             toml += textwrap.dedent(
                 f"""
-                control_plane_api = '{self.control_plane_api}'
+                [[pageservers]]
+                id={ps_id}
+                listen_pg_addr = 'localhost:{pageserver_port.pg}'
+                listen_http_addr = 'localhost:{pageserver_port.http}'
+                pg_auth_type = '{pg_auth_type}'
+                http_auth_type = '{http_auth_type}'
             """
             )
 
-        # Create a corresponding NeonPageserver object
-        self.pageserver = NeonPageserver(
-            self, port=pageserver_port, config_override=config.pageserver_config_override
-        )
-
+            # Create a corresponding NeonPageserver object
+            self.pageservers.append(
+                NeonPageserver(
+                    self,
+                    ps_id,
+                    port=pageserver_port,
+                    config_override=config.pageserver_config_override,
+                )
+            )
         # Create config and a Safekeeper object for each safekeeper
         for i in range(1, config.num_safekeepers + 1):
             port = SafekeeperPort(
@@ -798,25 +815,55 @@ class NeonEnv:
 
         if self.attachment_service is not None:
             self.attachment_service.start()
-        self.pageserver.start()
+
+        for pageserver in self.pageservers:
+            pageserver.start()
 
         for safekeeper in self.safekeepers:
             safekeeper.start()
 
+    @property
+    def pageserver(self) -> NeonPageserver:
+        """
+        For tests that are naive to multiple pageservers: give them the 1st in the list, and
+        assert that there is only one. Tests with multiple pageservers should always use
+        get_pageserver with an explicit ID.
+        """
+        assert len(self.pageservers) == 1
+        return self.pageservers[0]
+
+    def get_pageserver(self, id: Optional[int]) -> NeonPageserver:
+        """
+        Look up a pageserver by its node ID.
+
+        As a convenience for tests that do not use multiple pageservers, passing None
+        will yield the same default pageserver as `self.pageserver`.
+        """
+
+        if id is None:
+            return self.pageserver
+
+        for ps in self.pageservers:
+            if ps.id == id:
+                return ps
+
+        raise RuntimeError(f"Pageserver with ID {id} not found")
+
     def get_safekeeper_connstrs(self) -> str:
         """Get list of safekeeper endpoints suitable for safekeepers GUC"""
         return ",".join(f"localhost:{wa.port.pg}" for wa in self.safekeepers)
 
-    def timeline_dir(self, tenant_id: TenantId, timeline_id: TimelineId) -> Path:
-        """Get a timeline directory's path based on the repo directory of the test environment"""
-        return self.tenant_dir(tenant_id) / "timelines" / str(timeline_id)
-
-    def tenant_dir(
-        self,
-        tenant_id: TenantId,
+    def timeline_dir(
+        self, tenant_id: TenantId, timeline_id: TimelineId, pageserver_id: Optional[int] = None
     ) -> Path:
+        """Get a timeline directory's path based on the repo directory of the test environment"""
+        return (
+            self.tenant_dir(tenant_id, pageserver_id=pageserver_id) / "timelines" / str(timeline_id)
+        )
+
+    def tenant_dir(self, tenant_id: TenantId, pageserver_id: Optional[int] = None) -> Path:
         """Get a tenant directory's path based on the repo directory of the test environment"""
-        return self.repo_dir / "tenants" / str(tenant_id)
+        return self.get_pageserver(pageserver_id).workdir / "tenants" / str(tenant_id)
 
     def get_pageserver_version(self) -> str:
         bin_pageserver = str(self.neon_binpath / "pageserver")
@@ -984,6 +1031,7 @@ class AbstractNeonCli(abc.ABC):
         extra_env_vars: Optional[Dict[str, str]] = None,
         check_return_code=True,
         timeout=None,
+        local_binpath=False,
     ) -> "subprocess.CompletedProcess[str]":
         """
         Run the command with the specified arguments.
@@ -997,12 +1045,19 @@ class AbstractNeonCli(abc.ABC):
         >>> log.info(result.stdout)
 
         If `check_return_code`, on non-zero exit code logs failure and raises.
+
+        If `local_binpath` is true, then we are invoking a test utility
         """
 
         assert type(arguments) == list
         assert type(self.COMMAND) == str
 
-        bin_neon = str(self.env.neon_binpath / self.COMMAND)
+        if local_binpath:
+            # Test utility
+            bin_neon = str(self.env.neon_local_binpath / self.COMMAND)
+        else:
+            # Normal binary
+            bin_neon = str(self.env.neon_binpath / self.COMMAND)
 
         args = [bin_neon] + arguments
         log.info('Running command "{}"'.format(" ".join(args)))
@@ -1056,6 +1111,10 @@ class NeonCli(AbstractNeonCli):
 
     COMMAND = "neon_local"
 
+    def raw_cli(self, *args, **kwargs) -> subprocess.CompletedProcess[str]:
+        kwargs["local_binpath"] = True
+        return super().raw_cli(*args, **kwargs)
+
     def create_tenant(
         self,
         tenant_id: Optional[TenantId] = None,
@@ -1214,7 +1273,7 @@ class NeonCli(AbstractNeonCli):
             append_pageserver_param_overrides(
                 params_to_update=cmd,
                 remote_storage=storage,
-                pageserver_config_override=self.env.pageserver.config_override,
+                pageserver_config_override=self.env.pageserver_config_override,
             )
 
             s3_env_vars = None
@@ -1236,15 +1295,16 @@ class NeonCli(AbstractNeonCli):
 
     def pageserver_start(
         self,
+        id: int,
         overrides: Tuple[str, ...] = (),
         extra_env_vars: Optional[Dict[str, str]] = None,
     ) -> "subprocess.CompletedProcess[str]":
-        start_args = ["pageserver", "start", *overrides]
+        start_args = ["pageserver", "start", f"--id={id}", *overrides]
         storage = self.env.pageserver_remote_storage
         append_pageserver_param_overrides(
             params_to_update=start_args,
             remote_storage=storage,
-            pageserver_config_override=self.env.pageserver.config_override,
+            pageserver_config_override=self.env.pageserver_config_override,
         )
 
         if isinstance(storage, S3Storage):
@@ -1253,8 +1313,8 @@ class NeonCli(AbstractNeonCli):
 
         return self.raw_cli(start_args, extra_env_vars=extra_env_vars)
 
-    def pageserver_stop(self, immediate=False) -> "subprocess.CompletedProcess[str]":
-        cmd = ["pageserver", "stop"]
+    def pageserver_stop(self, id: int, immediate=False) -> "subprocess.CompletedProcess[str]":
+        cmd = ["pageserver", "stop", f"--id={id}"]
         if immediate:
             cmd.extend(["-m", "immediate"])
 
@@ -1295,6 +1355,7 @@ class NeonCli(AbstractNeonCli):
         tenant_id: Optional[TenantId] = None,
         hot_standby: bool = False,
         lsn: Optional[Lsn] = None,
+        pageserver_id: Optional[int] = None,
     ) -> "subprocess.CompletedProcess[str]":
         args = [
             "endpoint",
@@ -1316,6 +1377,8 @@ class NeonCli(AbstractNeonCli):
             args.append(endpoint_id)
         if hot_standby:
             args.extend(["--hot-standby", "true"])
+        if pageserver_id is not None:
+            args.extend(["--pageserver-id", str(pageserver_id)])
 
         res = self.raw_cli(args)
         res.check_returncode()
@@ -1331,6 +1394,7 @@ class NeonCli(AbstractNeonCli):
         lsn: Optional[Lsn] = None,
         branch_name: Optional[str] = None,
         remote_ext_config: Optional[str] = None,
+        pageserver_id: Optional[int] = None,
     ) -> "subprocess.CompletedProcess[str]":
         args = [
             "endpoint",
@@ -1353,6 +1417,8 @@ class NeonCli(AbstractNeonCli):
             args.extend(["--branch-name", branch_name])
         if endpoint_id is not None:
             args.append(endpoint_id)
+        if pageserver_id is not None:
+            args.extend(["--pageserver-id", str(pageserver_id)])
 
         storage = self.env.ext_remote_storage
         s3_env_vars = None
@@ -1452,9 +1518,12 @@ class NeonPageserver(PgProtocol):
 
     TEMP_FILE_SUFFIX = "___temp"
 
-    def __init__(self, env: NeonEnv, port: PageserverPort, config_override: Optional[str] = None):
+    def __init__(
+        self, env: NeonEnv, id: int, port: PageserverPort, config_override: Optional[str] = None
+    ):
         super().__init__(host="localhost", port=port.pg, user="cloud_admin")
         self.env = env
+        self.id = id
         self.running = False
         self.service_port = port
         self.config_override = config_override
@@ -1528,7 +1597,9 @@ class NeonPageserver(PgProtocol):
         """
         assert self.running is False
 
-        self.env.neon_cli.pageserver_start(overrides=overrides, extra_env_vars=extra_env_vars)
+        self.env.neon_cli.pageserver_start(
+            self.id, overrides=overrides, extra_env_vars=extra_env_vars
+        )
         self.running = True
         return self
 
@@ -1538,7 +1609,7 @@ class NeonPageserver(PgProtocol):
         Returns self.
         """
         if self.running:
-            self.env.neon_cli.pageserver_stop(immediate)
+            self.env.neon_cli.pageserver_stop(self.id, immediate)
             self.running = False
         return self
 
@@ -1564,8 +1635,12 @@ class NeonPageserver(PgProtocol):
             is_testing_enabled_or_skip=self.is_testing_enabled_or_skip,
         )
 
+    @property
+    def workdir(self) -> Path:
+        return Path(os.path.join(self.env.repo_dir, f"pageserver_{self.id}"))
+
     def assert_no_errors(self):
-        logfile = open(os.path.join(self.env.repo_dir, "pageserver.log"), "r")
+        logfile = open(os.path.join(self.workdir, "pageserver.log"), "r")
         error_or_warn = re.compile(r"\s(ERROR|WARN)")
         errors = []
         while True:
@@ -1588,7 +1663,7 @@ class NeonPageserver(PgProtocol):
 
     def log_contains(self, pattern: str) -> Optional[str]:
         """Check that the pageserver log contains a line that matches the given regex"""
-        logfile = open(os.path.join(self.env.repo_dir, "pageserver.log"), "r")
+        logfile = open(os.path.join(self.workdir, "pageserver.log"), "r")
 
         contains_re = re.compile(pattern)
 
@@ -1618,14 +1693,14 @@ class NeonPageserver(PgProtocol):
         if self.env.attachment_service is not None:
             response = requests.post(
                 f"{self.env.control_plane_api}/attach_hook",
-                json={"tenant_id": str(tenant_id), "pageserver_id": self.env.PAGESERVER_ID},
+                json={"tenant_id": str(tenant_id), "pageserver_id": self.id},
             )
             response.raise_for_status()
             generation = response.json()["gen"]
         else:
             generation = None
 
-        client = self.env.pageserver.http_client()
+        client = self.http_client()
         return client.tenant_attach(tenant_id, config, config_null, generation=generation)
 
 
@@ -2236,6 +2311,7 @@ class Endpoint(PgProtocol):
         hot_standby: bool = False,
         lsn: Optional[Lsn] = None,
         config_lines: Optional[List[str]] = None,
+        pageserver_id: Optional[int] = None,
     ) -> "Endpoint":
         """
         Create a new Postgres endpoint.
@@ -2257,6 +2333,7 @@ class Endpoint(PgProtocol):
             hot_standby=hot_standby,
             pg_port=self.pg_port,
             http_port=self.http_port,
+            pageserver_id=pageserver_id,
         )
         path = Path("endpoints") / self.endpoint_id / "pgdata"
         self.pgdata_dir = os.path.join(self.env.repo_dir, path)
@@ -2270,7 +2347,9 @@ class Endpoint(PgProtocol):
 
         return self
 
-    def start(self, remote_ext_config: Optional[str] = None) -> "Endpoint":
+    def start(
+        self, remote_ext_config: Optional[str] = None, pageserver_id: Optional[int] = None
+    ) -> "Endpoint":
         """
         Start the Postgres instance.
         Returns self.
@@ -2287,6 +2366,7 @@ class Endpoint(PgProtocol):
             tenant_id=self.tenant_id,
             safekeepers=self.active_safekeepers,
             remote_ext_config=remote_ext_config,
+            pageserver_id=pageserver_id,
         )
         self.running = True
 
@@ -2377,6 +2457,7 @@ class Endpoint(PgProtocol):
         lsn: Optional[Lsn] = None,
         config_lines: Optional[List[str]] = None,
         remote_ext_config: Optional[str] = None,
+        pageserver_id: Optional[int] = None,
     ) -> "Endpoint":
         """
         Create an endpoint, apply config, and start Postgres.
@@ -2391,6 +2472,7 @@ class Endpoint(PgProtocol):
             config_lines=config_lines,
             hot_standby=hot_standby,
             lsn=lsn,
+            pageserver_id=pageserver_id,
         ).start(remote_ext_config=remote_ext_config)
 
         log.info(f"Postgres startup took {time.time() - started_at} seconds")
@@ -2426,6 +2508,7 @@ class EndpointFactory:
         hot_standby: bool = False,
         config_lines: Optional[List[str]] = None,
         remote_ext_config: Optional[str] = None,
+        pageserver_id: Optional[int] = None,
     ) -> Endpoint:
         ep = Endpoint(
             self.env,
@@ -2443,6 +2526,7 @@ class EndpointFactory:
             config_lines=config_lines,
             lsn=lsn,
             remote_ext_config=remote_ext_config,
+            pageserver_id=pageserver_id,
         )
 
     def create(
@@ -2877,9 +2961,7 @@ def list_files_to_compare(pgdata_dir: Path) -> List[str]:
 
 # pg is the existing and running compute node, that we want to compare with a basebackup
 def check_restored_datadir_content(
-    test_output_dir: Path,
-    env: NeonEnv,
-    endpoint: Endpoint,
+    test_output_dir: Path, env: NeonEnv, endpoint: Endpoint, pageserver_id: Optional[int] = None
 ):
     # Get the timeline ID. We need it for the 'basebackup' command
     timeline_id = TimelineId(endpoint.safe_psql("SHOW neon.timeline_id")[0][0])
@@ -2904,7 +2986,7 @@ def check_restored_datadir_content(
     cmd = rf"""
         {psql_path}                                    \
             --no-psqlrc                                \
-            postgres://localhost:{env.pageserver.service_port.pg}  \
+            postgres://localhost:{env.get_pageserver(pageserver_id).service_port.pg}  \
             -c 'basebackup {endpoint.tenant_id} {timeline_id}'  \
          | tar -x -C {restored_dir_path}
     """
@@ -2954,19 +3036,32 @@ def check_restored_datadir_content(
 
 
 def wait_for_last_flush_lsn(
-    env: NeonEnv, endpoint: Endpoint, tenant: TenantId, timeline: TimelineId
+    env: NeonEnv,
+    endpoint: Endpoint,
+    tenant: TenantId,
+    timeline: TimelineId,
+    pageserver_id: Optional[int] = None,
 ) -> Lsn:
     """Wait for pageserver to catch up the latest flush LSN, returns the last observed lsn."""
+
     last_flush_lsn = Lsn(endpoint.safe_psql("SELECT pg_current_wal_flush_lsn()")[0][0])
-    return wait_for_last_record_lsn(env.pageserver.http_client(), tenant, timeline, last_flush_lsn)
+    return wait_for_last_record_lsn(
+        env.get_pageserver(pageserver_id).http_client(), tenant, timeline, last_flush_lsn
+    )
 
 
 def wait_for_wal_insert_lsn(
-    env: NeonEnv, endpoint: Endpoint, tenant: TenantId, timeline: TimelineId
+    env: NeonEnv,
+    endpoint: Endpoint,
+    tenant: TenantId,
+    timeline: TimelineId,
+    pageserver_id: Optional[int] = None,
 ) -> Lsn:
     """Wait for pageserver to catch up the latest flush LSN, returns the last observed lsn."""
     last_flush_lsn = Lsn(endpoint.safe_psql("SELECT pg_current_wal_insert_lsn()")[0][0])
-    return wait_for_last_record_lsn(env.pageserver.http_client(), tenant, timeline, last_flush_lsn)
+    return wait_for_last_record_lsn(
+        env.get_pageserver(pageserver_id).http_client(), tenant, timeline, last_flush_lsn
+    )
 
 
 def fork_at_current_lsn(
@@ -2986,15 +3081,21 @@ def fork_at_current_lsn(
 
 
 def last_flush_lsn_upload(
-    env: NeonEnv, endpoint: Endpoint, tenant_id: TenantId, timeline_id: TimelineId
+    env: NeonEnv,
+    endpoint: Endpoint,
+    tenant_id: TenantId,
+    timeline_id: TimelineId,
+    pageserver_id: Optional[int] = None,
 ) -> Lsn:
     """
     Wait for pageserver to catch to the latest flush LSN of given endpoint,
     checkpoint pageserver, and wait for it to be uploaded (remote_consistent_lsn
     reaching flush LSN).
     """
-    last_flush_lsn = wait_for_last_flush_lsn(env, endpoint, tenant_id, timeline_id)
-    ps_http = env.pageserver.http_client()
+    last_flush_lsn = wait_for_last_flush_lsn(
+        env, endpoint, tenant_id, timeline_id, pageserver_id=pageserver_id
+    )
+    ps_http = env.get_pageserver(pageserver_id).http_client()
     wait_for_last_record_lsn(ps_http, tenant_id, timeline_id, last_flush_lsn)
     # force a checkpoint to trigger upload
     ps_http.timeline_checkpoint(tenant_id, timeline_id)
diff --git a/test_runner/regress/test_broken_timeline.py b/test_runner/regress/test_broken_timeline.py
index 57e9413aa3..d0f180c866 100644
--- a/test_runner/regress/test_broken_timeline.py
+++ b/test_runner/regress/test_broken_timeline.py
@@ -44,14 +44,14 @@ def test_broken_timeline(neon_env_builder: NeonEnvBuilder):
     log.info(f"Timeline {tenant0}/{timeline0} is left intact")
 
     (tenant1, timeline1, pg1) = tenant_timelines[1]
-    metadata_path = f"{env.repo_dir}/tenants/{tenant1}/timelines/{timeline1}/metadata"
+    metadata_path = f"{env.pageserver.workdir}/tenants/{tenant1}/timelines/{timeline1}/metadata"
     f = open(metadata_path, "w")
     f.write("overwritten with garbage!")
     f.close()
     log.info(f"Timeline {tenant1}/{timeline1} got its metadata spoiled")
 
     (tenant2, timeline2, pg2) = tenant_timelines[2]
-    timeline_path = f"{env.repo_dir}/tenants/{tenant2}/timelines/{timeline2}/"
+    timeline_path = f"{env.pageserver.workdir}/tenants/{tenant2}/timelines/{timeline2}/"
     for filename in os.listdir(timeline_path):
         if filename.startswith("00000"):
             # Looks like a layer file. Remove it
@@ -61,7 +61,7 @@ def test_broken_timeline(neon_env_builder: NeonEnvBuilder):
     )
 
     (tenant3, timeline3, pg3) = tenant_timelines[3]
-    timeline_path = f"{env.repo_dir}/tenants/{tenant3}/timelines/{timeline3}/"
+    timeline_path = f"{env.pageserver.workdir}/tenants/{tenant3}/timelines/{timeline3}/"
     for filename in os.listdir(timeline_path):
         if filename.startswith("00000"):
             # Looks like a layer file. Corrupt it
@@ -135,7 +135,7 @@ def test_timeline_init_break_before_checkpoint(neon_simple_env: NeonEnv):
 
     tenant_id, _ = env.neon_cli.create_tenant()
 
-    timelines_dir = env.repo_dir / "tenants" / str(tenant_id) / "timelines"
+    timelines_dir = env.pageserver.workdir / "tenants" / str(tenant_id) / "timelines"
     old_tenant_timelines = env.neon_cli.list_timelines(tenant_id)
     initial_timeline_dirs = [d for d in timelines_dir.iterdir()]
 
@@ -145,8 +145,8 @@ def test_timeline_init_break_before_checkpoint(neon_simple_env: NeonEnv):
         _ = env.neon_cli.create_timeline("test_timeline_init_break_before_checkpoint", tenant_id)
 
     # Restart the page server
-    env.neon_cli.pageserver_stop(immediate=True)
-    env.neon_cli.pageserver_start()
+    env.pageserver.stop(immediate=True)
+    env.pageserver.start()
 
     # Creating the timeline didn't finish. The other timelines on tenant should still be present and work normally.
     new_tenant_timelines = env.neon_cli.list_timelines(tenant_id)
@@ -166,7 +166,7 @@ def test_timeline_create_break_after_uninit_mark(neon_simple_env: NeonEnv):
 
     tenant_id, _ = env.neon_cli.create_tenant()
 
-    timelines_dir = env.repo_dir / "tenants" / str(tenant_id) / "timelines"
+    timelines_dir = env.pageserver.workdir / "tenants" / str(tenant_id) / "timelines"
     old_tenant_timelines = env.neon_cli.list_timelines(tenant_id)
     initial_timeline_dirs = [d for d in timelines_dir.iterdir()]
 
diff --git a/test_runner/regress/test_close_fds.py b/test_runner/regress/test_close_fds.py
index 7059f3360e..ce9ecb3dc4 100644
--- a/test_runner/regress/test_close_fds.py
+++ b/test_runner/regress/test_close_fds.py
@@ -33,7 +33,7 @@ def test_lsof_pageserver_pid(neon_simple_env: NeonEnv):
     workload_thread = threading.Thread(target=start_workload, args=(), daemon=True)
     workload_thread.start()
 
-    path = os.path.join(env.repo_dir, "pageserver.pid")
+    path = os.path.join(env.pageserver.workdir, "pageserver.pid")
     lsof = lsof_path()
     while workload_thread.is_alive():
         res = subprocess.run(
diff --git a/test_runner/regress/test_compatibility.py b/test_runner/regress/test_compatibility.py
index 247b1d8bce..ad08a4b5ce 100644
--- a/test_runner/regress/test_compatibility.py
+++ b/test_runner/regress/test_compatibility.py
@@ -133,7 +133,6 @@ def test_backward_compatibility(
         prepare_snapshot(
             from_dir=compatibility_snapshot_dir,
             to_dir=test_output_dir / "compatibility_snapshot",
-            neon_binpath=neon_binpath,
             port_distributor=port_distributor,
         )
 
@@ -202,7 +201,6 @@ def test_forward_compatibility(
             from_dir=compatibility_snapshot_dir,
             to_dir=test_output_dir / "compatibility_snapshot",
             port_distributor=port_distributor,
-            neon_binpath=compatibility_neon_bin,
             pg_distrib_dir=compatibility_postgres_distrib_dir,
         )
 
@@ -234,7 +232,6 @@ def prepare_snapshot(
     from_dir: Path,
     to_dir: Path,
     port_distributor: PortDistributor,
-    neon_binpath: Path,
     pg_distrib_dir: Optional[Path] = None,
 ):
     assert from_dir.exists(), f"Snapshot '{from_dir}' doesn't exist"
@@ -246,6 +243,9 @@ def prepare_snapshot(
 
     repo_dir = to_dir / "repo"
 
+    snapshot_config_toml = repo_dir / "config"
+    snapshot_config = toml.load(snapshot_config_toml)
+
     # Remove old logs to avoid confusion in test artifacts
     for logfile in repo_dir.glob("**/*.log"):
         logfile.unlink()
@@ -259,31 +259,53 @@ def prepare_snapshot(
     os.mkdir(repo_dir / "endpoints")
 
     # Update paths and ports in config files
-    pageserver_toml = repo_dir / "pageserver.toml"
-    pageserver_config = toml.load(pageserver_toml)
-    pageserver_config["remote_storage"]["local_path"] = LocalFsStorage.component_path(
-        repo_dir, RemoteStorageUser.PAGESERVER
-    )
-    for param in ("listen_http_addr", "listen_pg_addr", "broker_endpoint"):
-        pageserver_config[param] = port_distributor.replace_with_new_port(pageserver_config[param])
+    legacy_pageserver_toml = repo_dir / "pageserver.toml"
+    legacy_bundle = os.path.exists(legacy_pageserver_toml)
 
-    # We don't use authentication in compatibility tests
-    # so just remove authentication related settings.
-    pageserver_config.pop("pg_auth_type", None)
-    pageserver_config.pop("http_auth_type", None)
-
-    if pg_distrib_dir:
-        pageserver_config["pg_distrib_dir"] = str(pg_distrib_dir)
-
-    with pageserver_toml.open("w") as f:
-        toml.dump(pageserver_config, f)
-
-    snapshot_config_toml = repo_dir / "config"
-    snapshot_config = toml.load(snapshot_config_toml)
-    for param in ("listen_http_addr", "listen_pg_addr"):
-        snapshot_config["pageserver"][param] = port_distributor.replace_with_new_port(
-            snapshot_config["pageserver"][param]
+    path_to_config: dict[Path, dict[Any, Any]] = {}
+    if legacy_bundle:
+        os.mkdir(repo_dir / "pageserver_1")
+        path_to_config[repo_dir / "pageserver_1" / "pageserver.toml"] = toml.load(
+            legacy_pageserver_toml
         )
+        os.remove(legacy_pageserver_toml)
+        os.rename(repo_dir / "tenants", repo_dir / "pageserver_1" / "tenants")
+    else:
+        for ps_conf in snapshot_config["pageservers"]:
+            config_path = repo_dir / f"pageserver_{ps_conf['id']}" / "pageserver.toml"
+            path_to_config[config_path] = toml.load(config_path)
+
+    # For each pageserver config, edit it and rewrite
+    for config_path, pageserver_config in path_to_config.items():
+        pageserver_config["remote_storage"]["local_path"] = LocalFsStorage.component_path(
+            repo_dir, RemoteStorageUser.PAGESERVER
+        )
+
+        for param in ("listen_http_addr", "listen_pg_addr", "broker_endpoint"):
+            pageserver_config[param] = port_distributor.replace_with_new_port(
+                pageserver_config[param]
+            )
+
+        # We don't use authentication in compatibility tests
+        # so just remove authentication related settings.
+        pageserver_config.pop("pg_auth_type", None)
+        pageserver_config.pop("http_auth_type", None)
+
+        if pg_distrib_dir:
+            pageserver_config["pg_distrib_dir"] = str(pg_distrib_dir)
+
+        with config_path.open("w") as f:
+            toml.dump(pageserver_config, f)
+
+    # neon_local config doesn't have to be backward compatible.  If we're using a dump from before
+    # it supported multiple pageservers, fix it up.
+    if "pageservers" not in snapshot_config:
+        snapshot_config["pageservers"] = [snapshot_config["pageserver"]]
+        del snapshot_config["pageserver"]
+
+    for param in ("listen_http_addr", "listen_pg_addr"):
+        for pageserver in snapshot_config["pageservers"]:
+            pageserver[param] = port_distributor.replace_with_new_port(pageserver[param])
     snapshot_config["broker"]["listen_addr"] = port_distributor.replace_with_new_port(
         snapshot_config["broker"]["listen_addr"]
     )
@@ -347,6 +369,9 @@ def check_neon_works(
     # Use the "target" binaries to launch the storage nodes
     config_target = config
     config_target.neon_binpath = neon_target_binpath
+    # We are using maybe-old binaries for neon services, but want to use current
+    # binaries for test utilities like neon_local
+    config_target.neon_local_binpath = neon_current_binpath
     cli_target = NeonCli(config_target)
 
     # And the current binaries to launch computes
@@ -379,7 +404,7 @@ def check_neon_works(
     # loosely based on https://github.com/neondatabase/cloud/wiki/Recovery-from-WAL
     tenant_id = snapshot_config["default_tenant_id"]
     timeline_id = dict(snapshot_config["branch_name_mappings"]["main"])[tenant_id]
-    pageserver_port = snapshot_config["pageserver"]["listen_http_addr"].split(":")[-1]
+    pageserver_port = snapshot_config["pageservers"][0]["listen_http_addr"].split(":")[-1]
     pageserver_http = PageserverHttpClient(
         port=pageserver_port,
         is_testing_enabled_or_skip=lambda: True,  # TODO: check if testing really enabled
diff --git a/test_runner/regress/test_import.py b/test_runner/regress/test_import.py
index ec1196eea6..1722d6bfe7 100644
--- a/test_runner/regress/test_import.py
+++ b/test_runner/regress/test_import.py
@@ -271,7 +271,7 @@ def _import(
     env.endpoints.stop_all()
     env.pageserver.stop()
 
-    dir_to_clear = Path(env.repo_dir) / "tenants"
+    dir_to_clear = Path(env.pageserver.workdir) / "tenants"
     shutil.rmtree(dir_to_clear)
     os.mkdir(dir_to_clear)
 
diff --git a/test_runner/regress/test_large_schema.py b/test_runner/regress/test_large_schema.py
index 72bf32fcd3..b6ac1aa41f 100644
--- a/test_runner/regress/test_large_schema.py
+++ b/test_runner/regress/test_large_schema.py
@@ -75,7 +75,7 @@ def test_large_schema(neon_env_builder: NeonEnvBuilder):
 
     # Check layer file sizes
     timeline_path = "{}/tenants/{}/timelines/{}/".format(
-        env.repo_dir, env.initial_tenant, env.initial_timeline
+        env.pageserver.workdir, env.initial_tenant, env.initial_timeline
     )
     for filename in os.listdir(timeline_path):
         if filename.startswith("00000"):
diff --git a/test_runner/regress/test_neon_cli.py b/test_runner/regress/test_neon_cli.py
index 9d24594cb6..1b3984583a 100644
--- a/test_runner/regress/test_neon_cli.py
+++ b/test_runner/regress/test_neon_cli.py
@@ -124,10 +124,14 @@ def test_cli_ipv4_listeners(neon_env_builder: NeonEnvBuilder):
 
 
 def test_cli_start_stop(neon_env_builder: NeonEnvBuilder):
+    """
+    Basic start/stop with default single-instance config for
+    safekeeper and pageserver
+    """
     env = neon_env_builder.init_start()
 
     # Stop default ps/sk
-    env.neon_cli.pageserver_stop()
+    env.neon_cli.pageserver_stop(env.pageserver.id)
     env.neon_cli.safekeeper_stop()
 
     # Default start
@@ -139,6 +143,38 @@ def test_cli_start_stop(neon_env_builder: NeonEnvBuilder):
     res.check_returncode()
 
 
+def test_cli_start_stop_multi(neon_env_builder: NeonEnvBuilder):
+    """
+    Basic start/stop with explicitly configured counts of pageserver
+    and safekeeper
+    """
+    neon_env_builder.num_pageservers = 2
+    neon_env_builder.num_safekeepers = 2
+    env = neon_env_builder.init_start()
+
+    env.neon_cli.pageserver_stop(env.BASE_PAGESERVER_ID)
+    env.neon_cli.pageserver_stop(env.BASE_PAGESERVER_ID + 1)
+
+    # Addressing a nonexistent ID throws
+    with pytest.raises(RuntimeError):
+        env.neon_cli.pageserver_stop(env.BASE_PAGESERVER_ID + 100)
+
+    # Using the single-pageserver shortcut property throws when there are multiple pageservers
+    with pytest.raises(AssertionError):
+        _drop = env.pageserver
+
+    env.neon_cli.safekeeper_stop(neon_env_builder.safekeepers_id_start + 1)
+    env.neon_cli.safekeeper_stop(neon_env_builder.safekeepers_id_start + 2)
+
+    # Default start
+    res = env.neon_cli.raw_cli(["start"])
+    res.check_returncode()
+
+    # Default stop
+    res = env.neon_cli.raw_cli(["stop"])
+    res.check_returncode()
+
+
 @skip_on_postgres(PgVersion.V14, reason="does not use postgres")
 @pytest.mark.skipif(
     os.environ.get("BUILD_TYPE") == "debug", reason="unit test for test support, either build works"
diff --git a/test_runner/regress/test_ondemand_download.py b/test_runner/regress/test_ondemand_download.py
index 4d5971cd11..259c6412ef 100644
--- a/test_runner/regress/test_ondemand_download.py
+++ b/test_runner/regress/test_ondemand_download.py
@@ -115,7 +115,7 @@ def test_ondemand_download_large_rel(
     env.pageserver.stop()
 
     # remove all the layer files
-    for layer in (Path(env.repo_dir) / "tenants").glob("*/timelines/*/*-*_*"):
+    for layer in (Path(env.pageserver.workdir) / "tenants").glob("*/timelines/*/*-*_*"):
         log.info(f"unlinking layer {layer}")
         layer.unlink()
 
@@ -237,7 +237,7 @@ def test_ondemand_download_timetravel(
     env.pageserver.stop()
 
     # remove all the layer files
-    for layer in (Path(env.repo_dir) / "tenants").glob("*/timelines/*/*-*_*"):
+    for layer in (Path(env.pageserver.workdir) / "tenants").glob("*/timelines/*/*-*_*"):
         log.info(f"unlinking layer {layer}")
         layer.unlink()
 
@@ -367,7 +367,7 @@ def test_download_remote_layers_api(
 
     # remove all the layer files
     # XXX only delete some of the layer files, to show that it really just downloads all the layers
-    for layer in (Path(env.repo_dir) / "tenants").glob("*/timelines/*/*-*_*"):
+    for layer in (Path(env.pageserver.workdir) / "tenants").glob("*/timelines/*/*-*_*"):
         log.info(f"unlinking layer {layer.name}")
         layer.unlink()
 
diff --git a/test_runner/regress/test_pageserver_api.py b/test_runner/regress/test_pageserver_api.py
index 28732872df..f5bcfd52f9 100644
--- a/test_runner/regress/test_pageserver_api.py
+++ b/test_runner/regress/test_pageserver_api.py
@@ -17,13 +17,13 @@ from fixtures.utils import wait_until
 def test_pageserver_init_node_id(
     neon_simple_env: NeonEnv, neon_binpath: Path, pg_distrib_dir: Path
 ):
-    repo_dir = neon_simple_env.repo_dir
-    pageserver_config = repo_dir / "pageserver.toml"
+    workdir = neon_simple_env.pageserver.workdir
+    pageserver_config = workdir / "pageserver.toml"
     pageserver_bin = neon_binpath / "pageserver"
 
     def run_pageserver(args):
         return subprocess.run(
-            [str(pageserver_bin), "-D", str(repo_dir), *args],
+            [str(pageserver_bin), "-D", str(workdir), *args],
             check=False,
             universal_newlines=True,
             stdout=subprocess.PIPE,
diff --git a/test_runner/regress/test_read_trace.py b/test_runner/regress/test_read_trace.py
index cae8ca3919..e6b3ccd7ec 100644
--- a/test_runner/regress/test_read_trace.py
+++ b/test_runner/regress/test_read_trace.py
@@ -35,5 +35,5 @@ def test_read_request_tracing(neon_env_builder: NeonEnvBuilder):
     # Stop postgres so we drop the connection and flush the traces
     endpoint.stop()
 
-    trace_path = env.repo_dir / "traces" / str(tenant_id) / str(timeline_id)
+    trace_path = env.pageserver.workdir / "traces" / str(tenant_id) / str(timeline_id)
     assert trace_path.exists()
diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py
index b01c4cc332..d0bb52336d 100644
--- a/test_runner/regress/test_remote_storage.py
+++ b/test_runner/regress/test_remote_storage.py
@@ -137,7 +137,7 @@ def test_remote_storage_backup_and_restore(
     env.endpoints.stop_all()
     env.pageserver.stop()
 
-    dir_to_clear = Path(env.repo_dir) / "tenants"
+    dir_to_clear = Path(env.pageserver.workdir) / "tenants"
     shutil.rmtree(dir_to_clear)
     os.mkdir(dir_to_clear)
 
@@ -353,7 +353,7 @@ def test_remote_storage_upload_queue_retries(
     env.pageserver.stop(immediate=True)
     env.endpoints.stop_all()
 
-    dir_to_clear = Path(env.repo_dir) / "tenants"
+    dir_to_clear = Path(env.pageserver.workdir) / "tenants"
     shutil.rmtree(dir_to_clear)
     os.mkdir(dir_to_clear)
 
@@ -488,7 +488,7 @@ def test_remote_timeline_client_calls_started_metric(
     env.pageserver.stop(immediate=True)
     env.endpoints.stop_all()
 
-    dir_to_clear = Path(env.repo_dir) / "tenants"
+    dir_to_clear = Path(env.pageserver.workdir) / "tenants"
     shutil.rmtree(dir_to_clear)
     os.mkdir(dir_to_clear)
 
diff --git a/test_runner/regress/test_tenant_conf.py b/test_runner/regress/test_tenant_conf.py
index 4fe6909433..d8a9e0390a 100644
--- a/test_runner/regress/test_tenant_conf.py
+++ b/test_runner/regress/test_tenant_conf.py
@@ -299,7 +299,7 @@ def test_creating_tenant_conf_after_attach(neon_env_builder: NeonEnvBuilder):
 
     # tenant is created with defaults, as in without config file
     (tenant_id, timeline_id) = env.neon_cli.create_tenant()
-    config_path = env.repo_dir / "tenants" / str(tenant_id) / "config"
+    config_path = env.pageserver.workdir / "tenants" / str(tenant_id) / "config"
     assert config_path.exists(), "config file is always initially created"
 
     http_client = env.pageserver.http_client()
diff --git a/test_runner/regress/test_tenant_delete.py b/test_runner/regress/test_tenant_delete.py
index 68188eb80a..e60b996a16 100644
--- a/test_runner/regress/test_tenant_delete.py
+++ b/test_runner/regress/test_tenant_delete.py
@@ -1,7 +1,6 @@
 import enum
 import os
 import shutil
-from pathlib import Path
 
 import pytest
 from fixtures.log_helper import log
@@ -367,7 +366,7 @@ def test_tenant_delete_is_resumed_on_attach(
     env.endpoints.stop_all()
     env.pageserver.stop()
 
-    dir_to_clear = Path(env.repo_dir) / "tenants"
+    dir_to_clear = env.pageserver.workdir / "tenants"
     shutil.rmtree(dir_to_clear)
     os.mkdir(dir_to_clear)
 
diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py
index 4e3c13123d..cdf72edc4d 100644
--- a/test_runner/regress/test_tenant_detach.py
+++ b/test_runner/regress/test_tenant_detach.py
@@ -289,7 +289,7 @@ def test_tenant_detach_smoke(neon_env_builder: NeonEnvBuilder):
     )
 
     # assert tenant exists on disk
-    assert (env.repo_dir / "tenants" / str(tenant_id)).exists()
+    assert (env.pageserver.workdir / "tenants" / str(tenant_id)).exists()
 
     endpoint = env.endpoints.create_start("main", tenant_id=tenant_id)
     # we rely upon autocommit after each statement
@@ -332,7 +332,7 @@ def test_tenant_detach_smoke(neon_env_builder: NeonEnvBuilder):
     log.info("gc thread returned")
 
     # check that nothing is left on disk for deleted tenant
-    assert not (env.repo_dir / "tenants" / str(tenant_id)).exists()
+    assert not (env.pageserver.workdir / "tenants" / str(tenant_id)).exists()
 
     with pytest.raises(
         expected_exception=PageserverApiException, match=f"NotFound: tenant {tenant_id}"
@@ -357,7 +357,7 @@ def test_tenant_detach_ignored_tenant(neon_simple_env: NeonEnv):
     )
 
     # assert tenant exists on disk
-    assert (env.repo_dir / "tenants" / str(tenant_id)).exists()
+    assert (env.pageserver.workdir / "tenants" / str(tenant_id)).exists()
 
     endpoint = env.endpoints.create_start("main", tenant_id=tenant_id)
     # we rely upon autocommit after each statement
@@ -386,7 +386,7 @@ def test_tenant_detach_ignored_tenant(neon_simple_env: NeonEnv):
     log.info("ignored tenant detached without error")
 
     # check that nothing is left on disk for deleted tenant
-    assert not (env.repo_dir / "tenants" / str(tenant_id)).exists()
+    assert not (env.pageserver.workdir / "tenants" / str(tenant_id)).exists()
 
     # assert the tenant does not exists in the Pageserver
     tenants_after_detach = [tenant["id"] for tenant in client.tenant_list()]
@@ -413,7 +413,7 @@ def test_tenant_detach_regular_tenant(neon_simple_env: NeonEnv):
     )
 
     # assert tenant exists on disk
-    assert (env.repo_dir / "tenants" / str(tenant_id)).exists()
+    assert (env.pageserver.workdir / "tenants" / str(tenant_id)).exists()
 
     endpoint = env.endpoints.create_start("main", tenant_id=tenant_id)
     # we rely upon autocommit after each statement
@@ -430,7 +430,7 @@ def test_tenant_detach_regular_tenant(neon_simple_env: NeonEnv):
     log.info("regular tenant detached without error")
 
     # check that nothing is left on disk for deleted tenant
-    assert not (env.repo_dir / "tenants" / str(tenant_id)).exists()
+    assert not (env.pageserver.workdir / "tenants" / str(tenant_id)).exists()
 
     # assert the tenant does not exists in the Pageserver
     tenants_after_detach = [tenant["id"] for tenant in client.tenant_list()]
@@ -531,7 +531,7 @@ def test_ignored_tenant_reattach(
     pageserver_http = env.pageserver.http_client()
 
     ignored_tenant_id, _ = env.neon_cli.create_tenant()
-    tenant_dir = env.repo_dir / "tenants" / str(ignored_tenant_id)
+    tenant_dir = env.pageserver.workdir / "tenants" / str(ignored_tenant_id)
     tenants_before_ignore = [tenant["id"] for tenant in pageserver_http.tenant_list()]
     tenants_before_ignore.sort()
     timelines_before_ignore = [
diff --git a/test_runner/regress/test_tenant_relocation.py b/test_runner/regress/test_tenant_relocation.py
index 88f921c2cd..a2b922bb84 100644
--- a/test_runner/regress/test_tenant_relocation.py
+++ b/test_runner/regress/test_tenant_relocation.py
@@ -561,7 +561,7 @@ def test_emergency_relocate_with_branches_slow_replay(
     # simpler than initializing a new one from scratch, but the effect on the single tenant
     # is the same.
     env.pageserver.stop(immediate=True)
-    shutil.rmtree(Path(env.repo_dir) / "tenants" / str(tenant_id))
+    shutil.rmtree(env.pageserver.workdir / "tenants" / str(tenant_id))
     env.pageserver.start()
 
     # This fail point will pause the WAL ingestion on the main branch, after the
@@ -709,7 +709,7 @@ def test_emergency_relocate_with_branches_createdb(
 
     # Kill the pageserver, remove the tenant directory, and restart
     env.pageserver.stop(immediate=True)
-    shutil.rmtree(Path(env.repo_dir) / "tenants" / str(tenant_id))
+    shutil.rmtree(env.pageserver.workdir / "tenants" / str(tenant_id))
     env.pageserver.start()
 
     # Wait before ingesting the WAL for CREATE DATABASE on the main branch. The original
diff --git a/test_runner/regress/test_tenants.py b/test_runner/regress/test_tenants.py
index 88094b8766..28f0e658a1 100644
--- a/test_runner/regress/test_tenants.py
+++ b/test_runner/regress/test_tenants.py
@@ -27,7 +27,7 @@ from prometheus_client.samples import Sample
 
 
 def test_tenant_creation_fails(neon_simple_env: NeonEnv):
-    tenants_dir = Path(neon_simple_env.repo_dir) / "tenants"
+    tenants_dir = Path(neon_simple_env.pageserver.workdir) / "tenants"
     initial_tenants = sorted(
         map(lambda t: t.split()[0], neon_simple_env.neon_cli.list_tenants().stdout.splitlines())
     )
@@ -322,7 +322,10 @@ def test_pageserver_with_empty_tenants(
     files_in_timelines_dir = sum(
         1
         for _p in Path.iterdir(
-            Path(env.repo_dir) / "tenants" / str(tenant_with_empty_timelines) / "timelines"
+            Path(env.pageserver.workdir)
+            / "tenants"
+            / str(tenant_with_empty_timelines)
+            / "timelines"
         )
     )
     assert (
@@ -334,7 +337,9 @@ def test_pageserver_with_empty_tenants(
     env.pageserver.stop()
 
     tenant_without_timelines_dir = env.initial_tenant
-    shutil.rmtree(Path(env.repo_dir) / "tenants" / str(tenant_without_timelines_dir) / "timelines")
+    shutil.rmtree(
+        Path(env.pageserver.workdir) / "tenants" / str(tenant_without_timelines_dir) / "timelines"
+    )
 
     env.pageserver.start()
 
diff --git a/test_runner/regress/test_tenants_with_remote_storage.py b/test_runner/regress/test_tenants_with_remote_storage.py
index bad090e83e..4aec4c6113 100644
--- a/test_runner/regress/test_tenants_with_remote_storage.py
+++ b/test_runner/regress/test_tenants_with_remote_storage.py
@@ -179,7 +179,9 @@ def test_tenants_attached_after_download(
 
     env.pageserver.stop()
 
-    timeline_dir = Path(env.repo_dir) / "tenants" / str(tenant_id) / "timelines" / str(timeline_id)
+    timeline_dir = (
+        Path(env.pageserver.workdir) / "tenants" / str(tenant_id) / "timelines" / str(timeline_id)
+    )
     local_layer_deleted = False
     for path in Path.iterdir(timeline_dir):
         if path.name.startswith("00000"):
diff --git a/test_runner/regress/test_timeline_delete.py b/test_runner/regress/test_timeline_delete.py
index cc770e5aa8..1e38e0534c 100644
--- a/test_runner/regress/test_timeline_delete.py
+++ b/test_runner/regress/test_timeline_delete.py
@@ -73,7 +73,11 @@ def test_timeline_delete(neon_simple_env: NeonEnv):
     )
 
     timeline_path = (
-        env.repo_dir / "tenants" / str(env.initial_tenant) / "timelines" / str(parent_timeline_id)
+        env.pageserver.workdir
+        / "tenants"
+        / str(env.initial_tenant)
+        / "timelines"
+        / str(parent_timeline_id)
     )
 
     with pytest.raises(
@@ -86,7 +90,11 @@ def test_timeline_delete(neon_simple_env: NeonEnv):
     assert exc.value.status_code == 412
 
     timeline_path = (
-        env.repo_dir / "tenants" / str(env.initial_tenant) / "timelines" / str(leaf_timeline_id)
+        env.pageserver.workdir
+        / "tenants"
+        / str(env.initial_tenant)
+        / "timelines"
+        / str(leaf_timeline_id)
     )
     assert timeline_path.exists()
 
@@ -406,7 +414,7 @@ def test_timeline_resurrection_on_attach(
     env.endpoints.stop_all()
     env.pageserver.stop()
 
-    dir_to_clear = Path(env.repo_dir) / "tenants"
+    dir_to_clear = Path(env.pageserver.workdir) / "tenants"
     shutil.rmtree(dir_to_clear)
     os.mkdir(dir_to_clear)
 
@@ -458,7 +466,11 @@ def test_timeline_delete_fail_before_local_delete(neon_env_builder: NeonEnvBuild
     )
 
     leaf_timeline_path = (
-        env.repo_dir / "tenants" / str(env.initial_tenant) / "timelines" / str(leaf_timeline_id)
+        env.pageserver.workdir
+        / "tenants"
+        / str(env.initial_tenant)
+        / "timelines"
+        / str(leaf_timeline_id)
     )
 
     ps_http.timeline_delete(env.initial_tenant, leaf_timeline_id)
@@ -907,7 +919,7 @@ def test_timeline_delete_resumed_on_attach(
     env.endpoints.stop_all()
     env.pageserver.stop()
 
-    dir_to_clear = Path(env.repo_dir) / "tenants"
+    dir_to_clear = Path(env.pageserver.workdir) / "tenants"
     shutil.rmtree(dir_to_clear)
     os.mkdir(dir_to_clear)
 
diff --git a/test_runner/regress/test_wal_restore.py b/test_runner/regress/test_wal_restore.py
index c97c69db23..a4f03be7a0 100644
--- a/test_runner/regress/test_wal_restore.py
+++ b/test_runner/regress/test_wal_restore.py
@@ -29,7 +29,7 @@ def test_wal_restore(
     endpoint.safe_psql("create table t as select generate_series(1,300000)")
     tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0])
     timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0])
-    env.neon_cli.pageserver_stop()
+    env.pageserver.stop()
     port = port_distributor.get_port()
     data_dir = test_output_dir / "pgsql.restored"
     with VanillaPostgres(
diff --git a/test_runner/regress/test_walredo_not_left_behind_on_detach.py b/test_runner/regress/test_walredo_not_left_behind_on_detach.py
index 4a47898935..155242ed15 100644
--- a/test_runner/regress/test_walredo_not_left_behind_on_detach.py
+++ b/test_runner/regress/test_walredo_not_left_behind_on_detach.py
@@ -27,7 +27,7 @@ def test_walredo_not_left_behind_on_detach(neon_env_builder: NeonEnvBuilder):
     env.pageserver.allowed_errors.append(".*NotFound: tenant.*")
     pageserver_http = env.pageserver.http_client()
 
-    pagserver_pid = int((env.repo_dir / "pageserver.pid").read_text())
+    pagserver_pid = int((env.pageserver.workdir / "pageserver.pid").read_text())
 
     assert_child_processes(pagserver_pid, wal_redo_present=False, defunct_present=False)
 
@@ -43,7 +43,7 @@ def test_walredo_not_left_behind_on_detach(neon_env_builder: NeonEnvBuilder):
     tenant_id, _ = env.neon_cli.create_tenant()
 
     # assert tenant exists on disk
-    assert (env.repo_dir / "tenants" / str(tenant_id)).exists()
+    assert (env.pageserver.workdir / "tenants" / str(tenant_id)).exists()
 
     endpoint = env.endpoints.create_start("main", tenant_id=tenant_id)
 
@@ -101,7 +101,7 @@ def test_walredo_not_left_behind_on_detach(neon_env_builder: NeonEnvBuilder):
         pytest.fail(f"could not detach tenant: {last_error}")
 
     # check that nothing is left on disk for deleted tenant
-    assert not (env.repo_dir / "tenants" / str(tenant_id)).exists()
+    assert not (env.pageserver.workdir / "tenants" / str(tenant_id)).exists()
 
     # Pageserver schedules kill+wait of the WAL redo process to the background runtime,
     # asynchronously to tenant detach. Cut it some slack to complete kill+wait before

From 028fbae161be63ddfc490a193e00d267e0d14eb6 Mon Sep 17 00:00:00 2001
From: Alexander Bayandin <alexander@neon.tech>
Date: Fri, 8 Sep 2023 16:28:09 +0100
Subject: [PATCH 42/61] Miscellaneous fixes for tests-related things (#5259)

## Problem

A bunch of fixes for different test-related things

## Summary of changes
- Fix test_runner/pg_clients (`subprocess_capture` return value has
changed)
- Do not run create-test-report if check-permissions failed for not
cancelled jobs
- Fix Code Coverage comment layout after flaky tests. Add another
healing "\n"
- test_compatibility: add an instruction for local run


Co-authored-by: Joonas Koivunen <joonas@neon.tech>
---
 .github/workflows/build_and_test.yml      |  2 +-
 scripts/comment-test-report.js            |  2 +-
 test_runner/pg_clients/test_pg_clients.py |  2 +-
 test_runner/regress/test_compatibility.py | 48 +++++++++++++++++++++++
 4 files changed, 51 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index bd57d5bd5a..18b8aaff07 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -444,7 +444,7 @@ jobs:
 
   create-test-report:
     needs: [ check-permissions, regress-tests, coverage-report, benchmarks ]
-    if: ${{ !cancelled() }}
+    if: ${{ !cancelled() && contains(fromJSON('["skipped", "success"]'), needs.check-permissions.result) }}
 
     runs-on: [ self-hosted, gen3, small ]
     container:
diff --git a/scripts/comment-test-report.js b/scripts/comment-test-report.js
index ce92534dea..07fb35cfc5 100755
--- a/scripts/comment-test-report.js
+++ b/scripts/comment-test-report.js
@@ -268,7 +268,7 @@ module.exports = async ({ github, context, fetch, report, coverage }) => {
             commentBody += "```\n"
         }
     } else {
-        commentBody += `#### Test coverage report is not avaibale\n`
+        commentBody += `\n#### Test coverage report is not available\n`
     }
 
     commentBody += autoupdateNotice
diff --git a/test_runner/pg_clients/test_pg_clients.py b/test_runner/pg_clients/test_pg_clients.py
index 7c20bac399..8381eac946 100644
--- a/test_runner/pg_clients/test_pg_clients.py
+++ b/test_runner/pg_clients/test_pg_clients.py
@@ -48,6 +48,6 @@ def test_pg_clients(test_output_dir: Path, remote_pg: RemotePostgres, client: st
     subprocess_capture(test_output_dir, build_cmd, check=True)
 
     run_cmd = [docker_bin, "run", "--rm", "--env-file", env_file, image_tag]
-    basepath = subprocess_capture(test_output_dir, run_cmd, check=True)
+    basepath, _, _ = subprocess_capture(test_output_dir, run_cmd, check=True)
 
     assert Path(f"{basepath}.stdout").read_text().strip() == "1"
diff --git a/test_runner/regress/test_compatibility.py b/test_runner/regress/test_compatibility.py
index ad08a4b5ce..b00cf983ed 100644
--- a/test_runner/regress/test_compatibility.py
+++ b/test_runner/regress/test_compatibility.py
@@ -40,6 +40,54 @@ from pytest import FixtureRequest
 # - prepare_snapshot copies the snapshot, cleans it up and makes it ready for the current version of Neon (replaces paths and ports in config files).
 # - check_neon_works performs the test itself, feel free to add more checks there.
 #
+#
+# How to run `test_backward_compatibility` locally:
+#
+#    export DEFAULT_PG_VERSION=15
+#    export BUILD_TYPE=release
+#    export CHECK_ONDISK_DATA_COMPATIBILITY=true
+#
+#    # Build previous version of binaries and create a data snapshot:
+#    rm -rf pg_install target
+#    git checkout <previous version>
+#    CARGO_BUILD_FLAGS="--features=testing" make -s -j`nproc`
+#    ./scripts/pytest -k test_create_snapshot
+#
+#    # Build current version of binaries
+#    rm -rf pg_install target
+#    git checkout <current version>
+#    CARGO_BUILD_FLAGS="--features=testing" make -s -j`nproc`
+#
+#    # Run backward compatibility test
+#    COMPATIBILITY_SNAPSHOT_DIR=test_output/compatibility_snapshot_pgv${DEFAULT_PG_VERSION} \
+#       ./scripts/pytest -k test_backward_compatibility
+#
+#
+# How to run `test_forward_compatibility` locally:
+#
+#    export DEFAULT_PG_VERSION=15
+#    export BUILD_TYPE=release
+#    export CHECK_ONDISK_DATA_COMPATIBILITY=true
+#
+#    # Build previous version of binaries and store them somewhere:
+#    rm -rf pg_install target
+#    git checkout <previous version>
+#    CARGO_BUILD_FLAGS="--features=testing" make -s -j`nproc`
+#    mkdir -p neon_previous/target
+#    cp -a target/${BUILD_TYPE} ./neon_previous/target/${BUILD_TYPE}
+#    cp -a pg_install ./neon_previous/pg_install
+#
+#    # Build current version of binaries and create a data snapshot:
+#    rm -rf pg_install target
+#    git checkout <current version>
+#    CARGO_BUILD_FLAGS="--features=testing" make -s -j`nproc`
+#    ./scripts/pytest -k test_create_snapshot
+#
+#    # Run forward compatibility test
+#    COMPATIBILITY_NEON_BIN=neon_previous/target/${BUILD_TYPE} \
+#    COMPATIBILITY_POSTGRES_DISTRIB_DIR=neon_previous/pg_install \
+#       ./scripts/pytest -k test_forward_compatibility
+#
 
 check_ondisk_data_compatibility_if_enabled = pytest.mark.skipif(
     os.environ.get("CHECK_ONDISK_DATA_COMPATIBILITY") is None,

From 66633ef2a9056ec27ad580a23c723140f7479a42 Mon Sep 17 00:00:00 2001
From: Joonas Koivunen <joonas@neon.tech>
Date: Fri, 8 Sep 2023 19:36:02 +0300
Subject: [PATCH 43/61] rust-toolchain: use 1.72.0, same as CI (#5256)

Switches everyone without an `rustup override` to 1.72.0.

Code changes required already done in #5255.
Depends on https://github.com/neondatabase/build/pull/65.
---
 rust-toolchain.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rust-toolchain.toml b/rust-toolchain.toml
index 0ce368ff9d..a21a293218 100644
--- a/rust-toolchain.toml
+++ b/rust-toolchain.toml
@@ -1,5 +1,5 @@
 [toolchain]
-channel = "1.71.0"
+channel = "1.72.0"
 profile = "default"
 # The default profile includes rustc, rust-std, cargo, rust-docs, rustfmt and clippy.
 # https://rust-lang.github.io/rustup/concepts/profiles.html

From 60050212e189cafa91dac6b412c663b8b518cee7 Mon Sep 17 00:00:00 2001
From: Heikki Linnakangas <heikki@neon.tech>
Date: Fri, 8 Sep 2023 19:40:29 +0300
Subject: [PATCH 44/61] Update rdkit to version 2023_03_03. (#5260)

It includes PostgreSQL 16 support.
---
 Dockerfile.compute-node | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node
index 09380a6d0d..79bce8c244 100644
--- a/Dockerfile.compute-node
+++ b/Dockerfile.compute-node
@@ -479,8 +479,8 @@ RUN apt-get update && \
         libfreetype6-dev
 
 ENV PATH "/usr/local/pgsql/bin/:/usr/local/pgsql/:$PATH"
-RUN wget https://github.com/rdkit/rdkit/archive/refs/tags/Release_2023_03_1.tar.gz -O rdkit.tar.gz && \
-    echo "db346afbd0ba52c843926a2a62f8a38c7b774ffab37eaf382d789a824f21996c rdkit.tar.gz" | sha256sum --check && \
+RUN wget https://github.com/rdkit/rdkit/archive/refs/tags/Release_2023_03_3.tar.gz -O rdkit.tar.gz && \
+    echo "bdbf9a2e6988526bfeb8c56ce3cdfe2998d60ac289078e2215374288185e8c8d rdkit.tar.gz" | sha256sum --check && \
     mkdir rdkit-src && cd rdkit-src && tar xvzf ../rdkit.tar.gz --strip-components=1 -C . && \
     cmake \
         -D RDK_BUILD_CAIRO_SUPPORT=OFF \

From 6f28da173723c7a9ac800d11205bab8f8e19c9ec Mon Sep 17 00:00:00 2001
From: Joonas Koivunen <joonas@neon.tech>
Date: Fri, 8 Sep 2023 20:27:00 +0300
Subject: [PATCH 45/61] fix: LocalFs root in test_compatibility is
 PosixPath('...') (#5261)

I forgot a `str(...)` conversion in #5243. This lead to log lines such
as:

```
Using fs root 'PosixPath('/tmp/test_output/test_backward_compatibility[debug-pg14]/compatibility_snapshot/repo/local_fs_remote_storage/pageserver')' as a remote storage
```

This surprisingly works, creating hierarchy of under current working
directory (`repo_dir` for tests):
- `PosixPath('`
  - `tmp` .. up until .. `local_fs_remote_storage`
    - `pageserver')`

It should not work but right now test_compatibility.py tests finds local
metadata and layers, which end up used. After #5172 when remote storage
is the source of truth it will no longer work.
---
 test_runner/regress/test_compatibility.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test_runner/regress/test_compatibility.py b/test_runner/regress/test_compatibility.py
index b00cf983ed..161662bc99 100644
--- a/test_runner/regress/test_compatibility.py
+++ b/test_runner/regress/test_compatibility.py
@@ -325,8 +325,8 @@ def prepare_snapshot(
 
     # For each pageserver config, edit it and rewrite
     for config_path, pageserver_config in path_to_config.items():
-        pageserver_config["remote_storage"]["local_path"] = LocalFsStorage.component_path(
-            repo_dir, RemoteStorageUser.PAGESERVER
+        pageserver_config["remote_storage"]["local_path"] = str(
+            LocalFsStorage.component_path(repo_dir, RemoteStorageUser.PAGESERVER)
         )
 
         for param in ("listen_http_addr", "listen_pg_addr", "broker_endpoint"):

From ba06ea26bbff4574efef0169769d6c08e531ddf4 Mon Sep 17 00:00:00 2001
From: Konstantin Knizhnik <knizhnik@garret.ru>
Date: Sat, 9 Sep 2023 17:51:16 +0300
Subject: [PATCH 46/61] Fix issues with reanabling LFC (#5209)

refer #5208

## Problem

See
https://neondb.slack.com/archives/C03H1K0PGKH/p1693938336062439?thread_ts=1693928260.704799&cid=C03H1K0PGKH

#5208 disable LFC forever in case of error. It is not good because the
problem causing this error (for example ENOSPC) can be resolved anti
will be nice to reenable it after fixing.

Also #5208 disables LFC locally in one backend. But other backends may
still see corrupted data.
It should not cause problems right now with "permission denied" error
because there should be no backend which is able to normally open LFC.
But in case of out-of-disk-space error, other backend can read corrupted
data.

## Summary of changes

1. Cleanup hash table after error to prevent access to stale or
corrupted data
2. Perform disk write under exclusive lock (hoping it will not affect
performance because usually write just copy data from user to system
space)
3. Use generations to prevent access to stale data in lfc_read

## Checklist before requesting a review

- [ ] I have performed a self-review of my code.
- [ ] If it is a core feature, I have added thorough tests.
- [ ] Do we need to implement analytics? if so did you add the relevant
metrics to the dashboard?
- [ ] If this PR requires public announcement, mark it with
/release-notes label and add several sentences in this section.

## Checklist before merging

- [ ] Do not forget to reformat commit message to not include the above
checklist

---------

Co-authored-by: Konstantin Knizhnik <knizhnik@neon.tech>
---
 pgxn/neon/file_cache.c | 152 ++++++++++++++++++++++++++---------------
 1 file changed, 98 insertions(+), 54 deletions(-)

diff --git a/pgxn/neon/file_cache.c b/pgxn/neon/file_cache.c
index a16b6bca2f..300c80e276 100644
--- a/pgxn/neon/file_cache.c
+++ b/pgxn/neon/file_cache.c
@@ -77,6 +77,7 @@ typedef struct FileCacheEntry
 
 typedef struct FileCacheControl
 {
+	uint64 generation; /* generation is needed to handle correct hash reenabling */
 	uint32 size; /* size of cache file in chunks */
 	uint32 used; /* number of used chunks */
 	dlist_head lru; /* double linked list for LRU replacement algorithm */
@@ -88,7 +89,6 @@ static LWLockId lfc_lock;
 static int   lfc_max_size;
 static int   lfc_size_limit;
 static int   lfc_free_space_watermark;
-static bool	lfc_disabled_by_failure = false;
 static char* lfc_path;
 static  FileCacheControl* lfc_ctl;
 static shmem_startup_hook_type prev_shmem_startup_hook;
@@ -97,10 +97,62 @@ static shmem_request_hook_type prev_shmem_request_hook;
 #endif
 static int   lfc_shrinking_factor; /* power of two by which local cache size will be shrinked when lfc_free_space_watermark is reached */
 
-#define DISABLE_LFC() (lfc_max_size = 0, lfc_disabled_by_failure = true, lfc_desc = -1)
-
 void FileCacheMonitorMain(Datum main_arg);
 
+/*
+ * Local file cache is mandatory and Neon can work without it.
+ * In case of any any errors with this cache, we should disable it but to not throw error.
+ * Also we should allow  re-enable it if source of failure (lack of disk space, permissions,...) is fixed.
+ * All cache content should be invalidated to avoid reading of stale or corrupted data
+ */
+static void
+lfc_disable(char const* op)
+{
+	HASH_SEQ_STATUS status;
+	FileCacheEntry* entry;
+
+	elog(WARNING, "Failed to %s local file cache at %s: %m, disabling local file cache", op, lfc_path);
+
+	if (lfc_desc > 0)
+		close(lfc_desc);
+
+	lfc_desc = -1;
+	lfc_size_limit = 0;
+
+	/* Invalidate hash */
+	LWLockAcquire(lfc_lock, LW_EXCLUSIVE);
+
+	hash_seq_init(&status, lfc_hash);
+	while ((entry = hash_seq_search(&status)) != NULL)
+	{
+		hash_search(lfc_hash, &entry->key, HASH_REMOVE, NULL);
+		memset(entry->bitmap, 0, sizeof entry->bitmap);
+	}
+	hash_seq_term(&status);
+	lfc_ctl->generation += 1;
+	lfc_ctl->size = 0;
+	lfc_ctl->used = 0;
+	dlist_init(&lfc_ctl->lru);
+
+	LWLockRelease(lfc_lock);
+}
+
+static bool
+lfc_ensure_opened(void)
+{
+	/* Open cache file if not done yet */
+	if (lfc_desc <= 0)
+	{
+		lfc_desc = BasicOpenFile(lfc_path, O_RDWR|O_CREAT);
+
+		if (lfc_desc < 0) {
+			lfc_disable("open");
+			return false;
+		}
+	}
+	return false;
+}
+
 static void
 lfc_shmem_startup(void)
 {
@@ -126,6 +178,7 @@ lfc_shmem_startup(void)
 								 lfc_size+1, lfc_size+1,
 								 &info,
 								 HASH_ELEM | HASH_BLOBS);
+		lfc_ctl->generation = 0;
 		lfc_ctl->size = 0;
 		lfc_ctl->used = 0;
 		dlist_init(&lfc_ctl->lru);
@@ -331,7 +384,7 @@ lfc_init(void)
 							   NULL,
 							   NULL);
 
-	if (lfc_max_size == 0 || lfc_disabled_by_failure)
+	if (lfc_max_size == 0)
 		return;
 
 	if (lfc_free_space_watermark != 0)
@@ -360,7 +413,7 @@ lfc_cache_contains(RelFileNode rnode, ForkNumber forkNum, BlockNumber blkno)
 	bool found;
 	uint32 hash;
 
-	if (lfc_size_limit == 0 || lfc_disabled_by_failure) /* fast exit if file cache is disabled */
+	if (lfc_size_limit == 0) /* fast exit if file cache is disabled */
 		return false;
 
 	tag.rnode = rnode;
@@ -387,7 +440,7 @@ lfc_evict(RelFileNode rnode, ForkNumber forkNum, BlockNumber blkno)
 	int chunk_offs = blkno & (BLOCKS_PER_CHUNK-1);
 	uint32 hash;
 
-	if (lfc_size_limit == 0 || lfc_disabled_by_failure) /* fast exit if file cache is disabled */
+	if (lfc_size_limit == 0) /* fast exit if file cache is disabled */
 		return;
 
 	INIT_BUFFERTAG(tag, rnode, forkNum, (blkno & ~(BLOCKS_PER_CHUNK-1)));
@@ -457,8 +510,13 @@ lfc_read(RelFileNode rnode, ForkNumber forkNum, BlockNumber blkno,
 	int chunk_offs = blkno & (BLOCKS_PER_CHUNK-1);
 	bool result = true;
 	uint32 hash;
+	uint64 generation;
+	uint32 entry_offset;
 
-	if (lfc_size_limit == 0 || lfc_disabled_by_failure) /* fast exit if file cache is disabled */
+	if (lfc_size_limit == 0) /* fast exit if file cache is disabled */
+		return false;
+
+	if (!lfc_ensure_opened())
 		return false;
 
 	tag.rnode = rnode;
@@ -477,37 +535,29 @@ lfc_read(RelFileNode rnode, ForkNumber forkNum, BlockNumber blkno,
 	/* Unlink entry from LRU list to pin it for the duration of IO operation */
 	if (entry->access_count++ == 0)
 		dlist_delete(&entry->lru_node);
+	generation = lfc_ctl->generation;
+	entry_offset = entry->offset;
+
 	LWLockRelease(lfc_lock);
 
-	/* Open cache file if not done yet */
-	if (lfc_desc <= 0)
+	rc = pread(lfc_desc, buffer, BLCKSZ, ((off_t)entry_offset*BLOCKS_PER_CHUNK + chunk_offs)*BLCKSZ);
+	if (rc != BLCKSZ)
 	{
-		lfc_desc = BasicOpenFile(lfc_path, O_RDWR|O_CREAT);
-
-		if (lfc_desc < 0) {
-			elog(LOG, "Failed to open file cache %s: %m", lfc_path);
-			DISABLE_LFC();
-			result = false;
-		}
-	}
-
-
-	if (lfc_desc > 0)
-	{
-		rc = pread(lfc_desc, buffer, BLCKSZ, ((off_t)entry->offset*BLOCKS_PER_CHUNK + chunk_offs)*BLCKSZ);
-		if (rc != BLCKSZ)
-		{
-			elog(INFO, "Failed to read file cache: %m");
-			DISABLE_LFC();
-			result = false;
-		}
+		lfc_disable("read");
+		return false;
 	}
 
 	/* Place entry to the head of LRU list */
 	LWLockAcquire(lfc_lock, LW_EXCLUSIVE);
-	Assert(entry->access_count > 0);
-	if (--entry->access_count == 0)
-		dlist_push_tail(&lfc_ctl->lru, &entry->lru_node);
+	if (lfc_ctl->generation == generation)
+	{
+		Assert(entry->access_count > 0);
+		if (--entry->access_count == 0)
+			dlist_push_tail(&lfc_ctl->lru, &entry->lru_node);
+	}
+	else
+		result = false;
+
 	LWLockRelease(lfc_lock);
 
 	return result;
@@ -528,7 +578,10 @@ lfc_write(RelFileNode rnode, ForkNumber forkNum, BlockNumber blkno,
 	int chunk_offs = blkno & (BLOCKS_PER_CHUNK-1);
 	uint32 hash;
 
-	if (lfc_size_limit == 0 || lfc_disabled_by_failure) /* fast exit if file cache is disabled */
+	if (lfc_size_limit == 0) /* fast exit if file cache is disabled */
+		return;
+
+	if (!lfc_ensure_opened())
 		return;
 
 	tag.rnode = rnode;
@@ -572,33 +625,22 @@ lfc_write(RelFileNode rnode, ForkNumber forkNum, BlockNumber blkno,
 		entry->access_count = 1;
 		memset(entry->bitmap, 0, sizeof entry->bitmap);
 	}
-	LWLockRelease(lfc_lock);
 
-	/* Open cache file if not done yet */
-	if (lfc_desc <= 0)
+	rc = pwrite(lfc_desc, buffer, BLCKSZ, ((off_t)entry->offset*BLOCKS_PER_CHUNK + chunk_offs)*BLCKSZ);
+	if (rc != BLCKSZ)
 	{
-		lfc_desc = BasicOpenFile(lfc_path, O_RDWR|O_CREAT);
-		if (lfc_desc < 0) {
-			elog(WARNING, "Failed to open file cache %s: %m, disabling file cache", lfc_path);
-			DISABLE_LFC(); /* disable file cache */
-		}
+		lfc_disable("write");
 	}
-	if (lfc_desc > 0)
+	else
 	{
-		rc = pwrite(lfc_desc, buffer, BLCKSZ, ((off_t)entry->offset*BLOCKS_PER_CHUNK + chunk_offs)*BLCKSZ);
-		if (rc != BLCKSZ)
-		{
-			elog(WARNING, "Failed to write file cache: %m, disabling file cache");
-			DISABLE_LFC(); /* disable file cache */
-		}
-	}
-	/* Place entry to the head of LRU list */
-	LWLockAcquire(lfc_lock, LW_EXCLUSIVE);
-	Assert(entry->access_count > 0);
-	if (--entry->access_count == 0)
-		dlist_push_tail(&lfc_ctl->lru, &entry->lru_node);
-	if (lfc_size_limit != 0)
+		/* Place entry to the head of LRU list */
+		Assert(entry->access_count > 0);
+		if (--entry->access_count == 0)
+			dlist_push_tail(&lfc_ctl->lru, &entry->lru_node);
+
 		entry->bitmap[chunk_offs >> 5] |= (1 << (chunk_offs & 31));
+	}
+
 	LWLockRelease(lfc_lock);
 }
 
@@ -699,6 +741,7 @@ local_cache_pages(PG_FUNCTION_ARGS)
 			for (int i = 0; i < BLOCKS_PER_CHUNK; i++)
 				n_pages += (entry->bitmap[i >> 5] & (1 << (i & 31))) != 0;
 		}
+		hash_seq_term(&status);
 		fctx->record = (LocalCachePagesRec *)
 			MemoryContextAllocHuge(CurrentMemoryContext,
 								   sizeof(LocalCachePagesRec) * n_pages);
@@ -737,6 +780,7 @@ local_cache_pages(PG_FUNCTION_ARGS)
 				}
 			}
 		}
+		hash_seq_term(&status);
 		Assert(n_pages == funcctx->max_calls);
 		LWLockRelease(lfc_lock);
 	}

From f64b338ce3b9e11944ef20c0497194c1ce56a167 Mon Sep 17 00:00:00 2001
From: Konstantin Knizhnik <knizhnik@garret.ru>
Date: Sat, 9 Sep 2023 21:51:04 +0300
Subject: [PATCH 47/61] Ingore DISK_FULL error when performing availability
 check for client (#5010)

See #5001

No space is what's expected if we're at size limit.
Of course if SK incorrectly returned "no space", the availability check
wouldn't fire.
But  users would notice such a bug quite soon anyways.
So  ignoring "no space" is the right trade-off.


## Problem

## Summary of changes

## Checklist before requesting a review

- [ ] I have performed a self-review of my code.
- [ ] If it is a core feature, I have added thorough tests.
- [ ] Do we need to implement analytics? if so did you add the relevant
metrics to the dashboard?
- [ ] If this PR requires public announcement, mark it with
/release-notes label and add several sentences in this section.

## Checklist before merging

- [ ] Do not forget to reformat commit message to not include the above
checklist

---------

Co-authored-by: Konstantin Knizhnik <knizhnik@neon.tech>
Co-authored-by: Joonas Koivunen <joonas@neon.tech>
---
 compute_tools/src/checker.rs | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/compute_tools/src/checker.rs b/compute_tools/src/checker.rs
index 6f52004fa8..d76eaad0a0 100644
--- a/compute_tools/src/checker.rs
+++ b/compute_tools/src/checker.rs
@@ -1,7 +1,7 @@
 use anyhow::{anyhow, Ok, Result};
 use postgres::Client;
 use tokio_postgres::NoTls;
-use tracing::{error, instrument};
+use tracing::{error, instrument, warn};
 
 use crate::compute::ComputeNode;
 
@@ -55,13 +55,24 @@ pub async fn check_writability(compute: &ComputeNode) -> Result<()> {
         ON CONFLICT (id) DO UPDATE
          SET updated_at = now();";
 
-    let result = client.simple_query(query).await?;
-
-    if result.len() != 1 {
-        return Err(anyhow::format_err!(
-            "expected 1 query result, but got {}",
-            result.len()
-        ));
+    match client.simple_query(query).await {
+        Result::Ok(result) => {
+            if result.len() != 1 {
+                return Err(anyhow::anyhow!(
+                    "expected 1 query results, but got {}",
+                    result.len()
+                ));
+            }
+        }
+        Err(err) => {
+            if let Some(state) = err.code() {
+                if state == &tokio_postgres::error::SqlState::DISK_FULL {
+                    warn!("Tenant disk is full");
+                    return Ok(());
+                }
+            }
+            return Err(err.into());
+        }
     }
 
     Ok(())

From 1ea93af56cbca1e91c6d78bee10ce4aad88c95d8 Mon Sep 17 00:00:00 2001
From: Alexander Bayandin <alexander@neon.tech>
Date: Sat, 9 Sep 2023 22:02:28 +0100
Subject: [PATCH 48/61] Create GitHub release from release tag (#5246)

## Problem

This PR creates a GitHub release from a release tag with an
autogenerated changelog: https://github.com/neondatabase/neon/releases

## Summary of changes
- Call GitHub API to create a release
---
 .github/workflows/build_and_test.yml | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 18b8aaff07..a1494ac40c 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -1081,13 +1081,27 @@ jobs:
           # Retry script for 5XX server errors: https://github.com/actions/github-script#retries
           retries: 5
           script: |
-            github.rest.git.createRef({
+            await github.rest.git.createRef({
               owner: context.repo.owner,
               repo: context.repo.repo,
               ref: "refs/tags/${{ needs.tag.outputs.build-tag }}",
               sha: context.sha,
             })
 
+      - name: Create GitHub release
+        if: github.ref_name == 'release'
+        uses: actions/github-script@v6
+        with:
+          # Retry script for 5XX server errors: https://github.com/actions/github-script#retries
+          retries: 5
+          script: |
+            await github.rest.repos.createRelease({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              tag_name: "${{ needs.tag.outputs.build-tag }}",
+              generate_release_notes: true,
+            })
+
   promote-compatibility-data:
     needs: [ check-permissions, promote-images, tag, regress-tests ]
     if: github.ref_name == 'release'

From 853552dcb4a09d52de79bcdd853112ce4a88f967 Mon Sep 17 00:00:00 2001
From: Em Sharnoff <sharnoff@neon.tech>
Date: Sun, 10 Sep 2023 10:15:53 -0700
Subject: [PATCH 49/61] vm-monitor: Don't include Args in top-level span
 (#5264)

It makes the logs too verbose.

ref https://neondb.slack.com/archives/C03F5SM1N02/p1694281232874719?thread_ts=1694272777.207109&cid=C03F5SM1N02
---
 libs/vm_monitor/src/lib.rs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/libs/vm_monitor/src/lib.rs b/libs/vm_monitor/src/lib.rs
index 2eee31c965..1cbfdc6ba6 100644
--- a/libs/vm_monitor/src/lib.rs
+++ b/libs/vm_monitor/src/lib.rs
@@ -178,14 +178,17 @@ pub async fn ws_handler(
 
 /// Starts the monitor. If startup fails or the monitor exits, an error will
 /// be logged and our internal state will be reset to allow for new connections.
-#[tracing::instrument(skip_all, fields(?args))]
+#[tracing::instrument(skip_all)]
 async fn start_monitor(
     ws: WebSocket,
     args: &Args,
     kill: broadcast::Receiver<()>,
     token: CancellationToken,
 ) {
-    info!("accepted new websocket connection -> starting monitor");
+    info!(
+        ?args,
+        "accepted new websocket connection -> starting monitor"
+    );
     let timeout = Duration::from_secs(4);
     let monitor = tokio::time::timeout(
         timeout,

From 1cac923af8142888570335e686fce385b959fca4 Mon Sep 17 00:00:00 2001
From: Em Sharnoff <sharnoff@neon.tech>
Date: Sun, 10 Sep 2023 10:33:53 -0700
Subject: [PATCH 50/61] vm-monitor: Rate-limit upscale requests (#5263)

Some VMs, when already scaled up as much as possible, end up spamming
the autoscaler-agent with upscale requests that will never be fulfilled.
If postgres is using memory greater than the cgroup's memory.high, it
can emit new memory.high events 1000 times per second, which... just
means unnecessary load on the rest of the system.

This changes the vm-monitor so that we skip sending upscale requests if
we already sent one within the last second, to avoid spamming the
autoscaler-agent. This matches previous behavior that the vm-informant
hand.
---
 libs/vm_monitor/src/runner.rs | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/libs/vm_monitor/src/runner.rs b/libs/vm_monitor/src/runner.rs
index 82055fda2e..8f904b879d 100644
--- a/libs/vm_monitor/src/runner.rs
+++ b/libs/vm_monitor/src/runner.rs
@@ -5,6 +5,7 @@
 //! all functionality.
 
 use std::sync::Arc;
+use std::time::{Duration, Instant};
 use std::{fmt::Debug, mem};
 
 use anyhow::{bail, Context};
@@ -36,6 +37,8 @@ pub struct Runner {
     /// by us vs the autoscaler-agent.
     counter: usize,
 
+    last_upscale_request_at: Option<Instant>,
+
     /// A signal to kill the main thread produced by `self.run()`. This is triggered
     /// when the server receives a new connection. When the thread receives the
     /// signal off this channel, it will gracefully shutdown.
@@ -99,6 +102,7 @@ impl Runner {
             cgroup: None,
             dispatcher,
             counter: 1, // NB: must be odd, see the comment about the field for more.
+            last_upscale_request_at: None,
             kill,
         };
 
@@ -397,6 +401,20 @@ impl Runner {
                     if request.is_none() {
                         bail!("failed to listen for upscale event from cgroup")
                     }
+
+                    // If it's been less than 1 second since the last time we requested upscaling,
+                    // ignore the event, to avoid spamming the agent (otherwise, this can happen
+                    // ~1k times per second).
+                    if let Some(t) = self.last_upscale_request_at {
+                        let elapsed = t.elapsed();
+                        if elapsed < Duration::from_secs(1) {
+                            info!(elapsed_millis = elapsed.as_millis(), "cgroup asked for upscale but too soon to forward the request, ignoring");
+                            continue;
+                        }
+                    }
+
+                    self.last_upscale_request_at = Some(Instant::now());
+
                     info!("cgroup asking for upscale; forwarding request");
                     self.counter += 2; // Increment, preserving parity (i.e. keep the
                                        // counter odd). See the field comment for more.

From 34e39645c4f18f9702bbc10029d809d0941b58e0 Mon Sep 17 00:00:00 2001
From: Alexander Bayandin <alexander@neon.tech>
Date: Sun, 10 Sep 2023 20:05:07 +0100
Subject: [PATCH 51/61] GitHub Workflows: add actionlint (#5265)

## Problem

Add a CI pipeline that checks GitHub Workflows with
https://github.com/rhysd/actionlint (it uses `shellcheck` for shell
scripts in steps)

To run it locally: `SHELLCHECK_OPTS=--exclude=SC2046,SC2086 actionlint`

## Summary of changes
- Add `.github/workflows/actionlint.yml`
- Fix actionlint warnings
---
 .github/actionlint.yml                    |  8 ++++++
 .github/workflows/actionlint.yml          | 31 +++++++++++++++++++++++
 .github/workflows/approved-for-ci-run.yml |  2 +-
 .github/workflows/benchmarking.yml        | 20 +++++++--------
 .github/workflows/build_and_test.yml      | 12 +++++----
 5 files changed, 57 insertions(+), 16 deletions(-)
 create mode 100644 .github/actionlint.yml
 create mode 100644 .github/workflows/actionlint.yml

diff --git a/.github/actionlint.yml b/.github/actionlint.yml
new file mode 100644
index 0000000000..fddd2f980d
--- /dev/null
+++ b/.github/actionlint.yml
@@ -0,0 +1,8 @@
+self-hosted-runner:
+  labels:
+    - gen3
+    - large
+    - small
+    - us-east-2
+config-variables:
+  - SLACK_UPCOMING_RELEASE_CHANNEL_ID
diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml
new file mode 100644
index 0000000000..584828c1d0
--- /dev/null
+++ b/.github/workflows/actionlint.yml
@@ -0,0 +1,31 @@
+name: Lint GitHub Workflows
+
+on:
+  push:
+    branches:
+      - main
+      - release
+    paths:
+      - '.github/workflows/*.ya?ml'
+  pull_request:
+    paths:
+      - '.github/workflows/*.ya?ml'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  actionlint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: reviewdog/action-actionlint@v1
+        env:
+          # SC2046 - Quote this to prevent word splitting. - https://www.shellcheck.net/wiki/SC2046
+          # SC2086 - Double quote to prevent globbing and word splitting. - https://www.shellcheck.net/wiki/SC2086
+          SHELLCHECK_OPTS: --exclude=SC2046,SC2086
+        with:
+          fail_on_error: true
+          filter_mode: nofilter
+          level: error
diff --git a/.github/workflows/approved-for-ci-run.yml b/.github/workflows/approved-for-ci-run.yml
index bcd042e608..17c42db0d2 100644
--- a/.github/workflows/approved-for-ci-run.yml
+++ b/.github/workflows/approved-for-ci-run.yml
@@ -96,7 +96,7 @@ jobs:
       - run: |
           HEAD="ci-run/pr-${PR_NUMBER}"
 
-          CLOSED="$(gh pr --repo '${GITHUB_REPOSITORY}' list --head '${HEAD}' --json 'closed' --jq '.[].closed')"
+          CLOSED="$(gh pr --repo ${GITHUB_REPOSITORY} list --head ${HEAD} --json 'closed' --jq '.[].closed')"
           if [ "${CLOSED}" != "false" ]; then
             gh pr --repo "${GITHUB_REPOSITORY}" close "ci-run/pr-${{ github.event.pull_request.number }}" --delete-branch
           fi
diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml
index 464b764d1a..e2f15d96db 100644
--- a/.github/workflows/benchmarking.yml
+++ b/.github/workflows/benchmarking.yml
@@ -137,11 +137,11 @@ jobs:
         }'
 
         if [ "$(date +%A)" = "Saturday" ]; then
-          matrix=$(echo $matrix | jq '.include += [{ "platform": "rds-postgres", "db_size": "10gb"},
+          matrix=$(echo "$matrix" | jq '.include += [{ "platform": "rds-postgres", "db_size": "10gb"},
                                                    { "platform": "rds-aurora",   "db_size": "50gb"}]')
         fi
 
-        echo "matrix=$(echo $matrix | jq --compact-output '.')" >> $GITHUB_OUTPUT
+        echo "matrix=$(echo "$matrix" | jq --compact-output '.')" >> $GITHUB_OUTPUT
 
     - name: Generate matrix for OLAP benchmarks
       id: olap-compare-matrix
@@ -153,11 +153,11 @@ jobs:
         }'
 
         if [ "$(date +%A)" = "Saturday" ]; then
-          matrix=$(echo $matrix | jq '.include += [{ "platform": "rds-postgres" },
+          matrix=$(echo "$matrix" | jq '.include += [{ "platform": "rds-postgres" },
                                                    { "platform": "rds-aurora"   }]')
         fi
 
-        echo "matrix=$(echo $matrix | jq --compact-output '.')" >> $GITHUB_OUTPUT
+        echo "matrix=$(echo "$matrix" | jq --compact-output '.')" >> $GITHUB_OUTPUT
 
     - name: Generate matrix for TPC-H benchmarks
       id: tpch-compare-matrix
@@ -172,11 +172,11 @@ jobs:
         }'
 
         if [ "$(date +%A)" = "Saturday" ]; then
-          matrix=$(echo $matrix | jq '.include += [{ "platform": "rds-postgres", "scale": "10" },
+          matrix=$(echo "$matrix" | jq '.include += [{ "platform": "rds-postgres", "scale": "10" },
                                                    { "platform": "rds-aurora",   "scale": "10" }]')
         fi
 
-        echo "matrix=$(echo $matrix | jq --compact-output '.')" >> $GITHUB_OUTPUT
+        echo "matrix=$(echo "$matrix" | jq --compact-output '.')" >> $GITHUB_OUTPUT
 
   pgbench-compare:
     needs: [ generate-matrices ]
@@ -254,7 +254,7 @@ jobs:
         echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
 
         QUERY="SELECT version();"
-        if [ "${PLATFORM}" = "neon"* ]; then
+        if [[ "${PLATFORM}" = "neon"* ]]; then
           QUERY="${QUERY} SHOW neon.tenant_id; SHOW neon.timeline_id;"
         fi
         psql ${CONNSTR} -c "${QUERY}"
@@ -383,7 +383,7 @@ jobs:
         echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
 
         QUERY="SELECT version();"
-        if [ "${PLATFORM}" = "neon"* ]; then
+        if [[ "${PLATFORM}" = "neon"* ]]; then
           QUERY="${QUERY} SHOW neon.tenant_id; SHOW neon.timeline_id;"
         fi
         psql ${CONNSTR} -c "${QUERY}"
@@ -487,7 +487,7 @@ jobs:
         echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
 
         QUERY="SELECT version();"
-        if [ "${PLATFORM}" = "neon"* ]; then
+        if [[ "${PLATFORM}" = "neon"* ]]; then
           QUERY="${QUERY} SHOW neon.tenant_id; SHOW neon.timeline_id;"
         fi
         psql ${CONNSTR} -c "${QUERY}"
@@ -577,7 +577,7 @@ jobs:
         echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
 
         QUERY="SELECT version();"
-        if [ "${PLATFORM}" = "neon"* ]; then
+        if [[ "${PLATFORM}" = "neon"* ]]; then
           QUERY="${QUERY} SHOW neon.tenant_id; SHOW neon.timeline_id;"
         fi
         psql ${CONNSTR} -c "${QUERY}"
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index a1494ac40c..898d19e7d4 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -254,10 +254,12 @@ jobs:
             cov_prefix=""
             CARGO_FLAGS="--locked --release"
           fi
-          echo "cov_prefix=${cov_prefix}" >> $GITHUB_ENV
-          echo "CARGO_FEATURES=${CARGO_FEATURES}" >> $GITHUB_ENV
-          echo "CARGO_FLAGS=${CARGO_FLAGS}" >> $GITHUB_ENV
-          echo "CARGO_HOME=${GITHUB_WORKSPACE}/.cargo" >> $GITHUB_ENV
+          {
+            echo "cov_prefix=${cov_prefix}"
+            echo "CARGO_FEATURES=${CARGO_FEATURES}"
+            echo "CARGO_FLAGS=${CARGO_FLAGS}"
+            echo "CARGO_HOME=${GITHUB_WORKSPACE}/.cargo"
+          } >> $GITHUB_ENV
 
       # Disabled for now
       # Don't include the ~/.cargo/registry/src directory. It contains just
@@ -1009,7 +1011,7 @@ jobs:
 
           last_status="" # a variable to carry the last status of the "build-and-upload-extensions" context
 
-          for ((i=0; i <= $TIMEOUT; i+=$INTERVAL)); do
+          for ((i=0; i <= TIMEOUT; i+=INTERVAL)); do
             sleep $INTERVAL
 
             # Get statuses for the latest commit in the PR / branch

From 15fd188fd693b8f2298b74cf9f378ea94865fa91 Mon Sep 17 00:00:00 2001
From: Alexander Bayandin <alexander@neon.tech>
Date: Sun, 10 Sep 2023 20:06:10 +0100
Subject: [PATCH 52/61] Fix GitHub Autocomment for `ci-run/pr`s (#5268)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Problem

When PR `ci-run/pr-*` is created the GitHub Autocomment with test
results are supposed to be posted to the original PR, currently, this
doesn't work.

I created this PR from a personal fork to debug and fix the issue.

## Summary of changes
- `scripts/comment-test-report.js`: use `pull_request.head` instead of
`pull_request.base` 🤦
---
 scripts/comment-test-report.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/comment-test-report.js b/scripts/comment-test-report.js
index 07fb35cfc5..89befda71f 100755
--- a/scripts/comment-test-report.js
+++ b/scripts/comment-test-report.js
@@ -210,7 +210,7 @@ module.exports = async ({ github, context, fetch, report, coverage }) => {
     // Which PR to comment (for ci-run/pr-* it will comment the parent PR, not the ci-run/pr-* PR)
     let prToComment
     if (isPullRequest) {
-        const branchName = context.payload.pull_request.base.ref.replace(/^refs\/heads\//, "")
+        const branchName = context.payload.pull_request.head.ref.replace(/^refs\/heads\//, "")
         const match = branchName.match(/ci-run\/pr-(?<prNumber>\d+)/)?.groups
         if (match) {
             ({ prNumber } = match)

From d33e1b1b242f0a1eef1d1675d39b28bf80a0d572 Mon Sep 17 00:00:00 2001
From: Alexander Bayandin <alexander@neon.tech>
Date: Sun, 10 Sep 2023 20:12:38 +0100
Subject: [PATCH 53/61] approved-for-ci-run.yml: use token to checkout the repo
 (#5266)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Problem

Another thing I overlooked regarding'approved-for-ci-run`:
- When we create a PR, the action is associated with @vipvap and this
triggers the pipeline — this is good.
- When we update the PR by force-pushing to the branch, the action is
associated with @github-actions, which doesn't trigger a pipeline — this
is bad.

Initially spotted in #5239 / #5211
([link](https://github.com/neondatabase/neon/actions/runs/6122249456/job/16633919558?pr=5239))
— `check-permissions` should not fail.


## Summary of changes
- Use `CI_ACCESS_TOKEN` to check out the repo (I expect this token will
be reused in the following `git push`)
---
 .github/workflows/approved-for-ci-run.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/approved-for-ci-run.yml b/.github/workflows/approved-for-ci-run.yml
index 17c42db0d2..e1c491b67c 100644
--- a/.github/workflows/approved-for-ci-run.yml
+++ b/.github/workflows/approved-for-ci-run.yml
@@ -54,6 +54,7 @@ jobs:
       - uses: actions/checkout@v3
         with:
           ref: main
+          token: ${{ secrets.CI_ACCESS_TOKEN }}
 
       - run: gh pr checkout "${PR_NUMBER}"
 

From 999fe668e70a11c83fb43ba0f5e07c1a4f36845d Mon Sep 17 00:00:00 2001
From: Rahul Modpur <rmodpur2@gmail.com>
Date: Mon, 11 Sep 2023 01:29:51 +0530
Subject: [PATCH 54/61] Ack tenant detach before local files are deleted
 (#5211)

## Problem

Detaching a tenant can involve many thousands of local filesystem
metadata writes, but the control plane would benefit from us not
blocking detach/delete responses on these.

## Summary of changes

After rename of local tenant directory ack tenant detach and delete
tenant directory in background

#5183

---------

Signed-off-by: Rahul Modpur <rmodpur2@gmail.com>
---
 pageserver/src/tenant/mgr.rs | 58 +++++++++++++++++++++++++-----------
 1 file changed, 41 insertions(+), 17 deletions(-)

diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs
index b92c1f69c3..0cf5c36b87 100644
--- a/pageserver/src/tenant/mgr.rs
+++ b/pageserver/src/tenant/mgr.rs
@@ -1,9 +1,10 @@
 //! This module acts as a switchboard to access different repositories managed by this
 //! page server.
 
+use rand::{distributions::Alphanumeric, Rng};
 use std::collections::{hash_map, HashMap};
 use std::ffi::OsStr;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use tokio::fs;
 
@@ -70,6 +71,11 @@ impl TenantsMap {
 /// This is pageserver-specific, as it relies on future processes after a crash to check
 /// for TEMP_FILE_SUFFIX when loading things.
 async fn safe_remove_tenant_dir_all(path: impl AsRef<Path>) -> std::io::Result<()> {
+    let tmp_path = safe_rename_tenant_dir(path).await?;
+    fs::remove_dir_all(tmp_path).await
+}
+
+async fn safe_rename_tenant_dir(path: impl AsRef<Path>) -> std::io::Result<PathBuf> {
     let parent = path
         .as_ref()
         .parent()
@@ -79,11 +85,16 @@ async fn safe_remove_tenant_dir_all(path: impl AsRef<Path>) -> std::io::Result<(
             std::io::ErrorKind::InvalidInput,
             "Path must be absolute",
         ))?;
-
-    let tmp_path = path_with_suffix_extension(&path, TEMP_FILE_SUFFIX);
+    let rand_suffix = rand::thread_rng()
+        .sample_iter(&Alphanumeric)
+        .take(8)
+        .map(char::from)
+        .collect::<String>()
+        + TEMP_FILE_SUFFIX;
+    let tmp_path = path_with_suffix_extension(&path, &rand_suffix);
     fs::rename(&path, &tmp_path).await?;
     fs::File::open(parent).await?.sync_all().await?;
-    fs::remove_dir_all(tmp_path).await
+    Ok(tmp_path)
 }
 
 static TENANTS: Lazy<RwLock<TenantsMap>> = Lazy::new(|| RwLock::new(TenantsMap::Initializing));
@@ -551,7 +562,24 @@ pub async fn detach_tenant(
     tenant_id: TenantId,
     detach_ignored: bool,
 ) -> Result<(), TenantStateError> {
-    detach_tenant0(conf, &TENANTS, tenant_id, detach_ignored).await
+    let tmp_path = detach_tenant0(conf, &TENANTS, tenant_id, detach_ignored).await?;
+    // Although we are cleaning up the tenant, this task is not meant to be bound by the lifetime of the tenant in memory.
+    // After a tenant is detached, there are no more task_mgr tasks for that tenant_id.
+    let task_tenant_id = None;
+    task_mgr::spawn(
+        task_mgr::BACKGROUND_RUNTIME.handle(),
+        TaskKind::MgmtRequest,
+        task_tenant_id,
+        None,
+        "tenant_files_delete",
+        false,
+        async move {
+            fs::remove_dir_all(tmp_path.as_path())
+                .await
+                .with_context(|| format!("tenant directory {:?} deletion", tmp_path))
+        },
+    );
+    Ok(())
 }
 
 async fn detach_tenant0(
@@ -559,20 +587,16 @@ async fn detach_tenant0(
     tenants: &tokio::sync::RwLock<TenantsMap>,
     tenant_id: TenantId,
     detach_ignored: bool,
-) -> Result<(), TenantStateError> {
-    let local_files_cleanup_operation = |tenant_id_to_clean| async move {
+) -> Result<PathBuf, TenantStateError> {
+    let tenant_dir_rename_operation = |tenant_id_to_clean| async move {
         let local_tenant_directory = conf.tenant_path(&tenant_id_to_clean);
-        safe_remove_tenant_dir_all(&local_tenant_directory)
+        safe_rename_tenant_dir(&local_tenant_directory)
             .await
-            .with_context(|| {
-                format!("local tenant directory {local_tenant_directory:?} removal")
-            })?;
-        Ok(())
+            .with_context(|| format!("local tenant directory {local_tenant_directory:?} rename"))
     };
 
     let removal_result =
-        remove_tenant_from_memory(tenants, tenant_id, local_files_cleanup_operation(tenant_id))
-            .await;
+        remove_tenant_from_memory(tenants, tenant_id, tenant_dir_rename_operation(tenant_id)).await;
 
     // Ignored tenants are not present in memory and will bail the removal from memory operation.
     // Before returning the error, check for ignored tenant removal case — we only need to clean its local files then.
@@ -580,10 +604,10 @@ async fn detach_tenant0(
         let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_id);
         if tenant_ignore_mark.exists() {
             info!("Detaching an ignored tenant");
-            local_files_cleanup_operation(tenant_id)
+            let tmp_path = tenant_dir_rename_operation(tenant_id)
                 .await
-                .with_context(|| format!("Ignored tenant {tenant_id} local files cleanup"))?;
-            return Ok(());
+                .with_context(|| format!("Ignored tenant {tenant_id} local directory rename"))?;
+            return Ok(tmp_path);
         }
     }
 

From a55a78a4535179d0241c49fd8cb3fdd9a0a5f045 Mon Sep 17 00:00:00 2001
From: Joonas Koivunen <joonas@neon.tech>
Date: Mon, 11 Sep 2023 11:42:49 +0300
Subject: [PATCH 55/61] Misc test flakyness fixes (#5233)

Assorted flakyness fixes from #5198, might not be flaky on `main`.

Migrate some tests using neon_simple_env to just neon_env_builder and
using initial_tenant to make flakyness understanding easier. (Did not
understand the flakyness of
`test_timeline_create_break_after_uninit_mark`.)

`test_download_remote_layers_api` is flaky because we have no atomic
"wait for WAL, checkpoint, wait for upload and do not receive any more
WAL".

`test_tenant_size` fixes are just boilerplate which should had always
existed; we should wait for the tenant to be active. similarly for
`test_timeline_delete`.

`test_timeline_size_post_checkpoint` fails often for me with reading
zero from metrics. Give it a few attempts.
---
 test_runner/regress/test_broken_timeline.py   | 12 ++++++------
 test_runner/regress/test_ondemand_download.py | 12 +++++++++++-
 test_runner/regress/test_tenant_size.py       |  9 ++++++++-
 test_runner/regress/test_timeline_delete.py   |  2 ++
 test_runner/regress/test_timeline_size.py     | 11 +++++++----
 5 files changed, 34 insertions(+), 12 deletions(-)

diff --git a/test_runner/regress/test_broken_timeline.py b/test_runner/regress/test_broken_timeline.py
index d0f180c866..c446f865cb 100644
--- a/test_runner/regress/test_broken_timeline.py
+++ b/test_runner/regress/test_broken_timeline.py
@@ -122,8 +122,8 @@ def test_create_multiple_timelines_parallel(neon_simple_env: NeonEnv):
             future.result()
 
 
-def test_timeline_init_break_before_checkpoint(neon_simple_env: NeonEnv):
-    env = neon_simple_env
+def test_timeline_init_break_before_checkpoint(neon_env_builder: NeonEnvBuilder):
+    env = neon_env_builder.init_start()
     pageserver_http = env.pageserver.http_client()
 
     env.pageserver.allowed_errors.extend(
@@ -133,7 +133,7 @@ def test_timeline_init_break_before_checkpoint(neon_simple_env: NeonEnv):
         ]
     )
 
-    tenant_id, _ = env.neon_cli.create_tenant()
+    tenant_id = env.initial_tenant
 
     timelines_dir = env.pageserver.workdir / "tenants" / str(tenant_id) / "timelines"
     old_tenant_timelines = env.neon_cli.list_timelines(tenant_id)
@@ -160,11 +160,11 @@ def test_timeline_init_break_before_checkpoint(neon_simple_env: NeonEnv):
     ), "pageserver should clean its temp timeline files on timeline creation failure"
 
 
-def test_timeline_create_break_after_uninit_mark(neon_simple_env: NeonEnv):
-    env = neon_simple_env
+def test_timeline_create_break_after_uninit_mark(neon_env_builder: NeonEnvBuilder):
+    env = neon_env_builder.init_start()
     pageserver_http = env.pageserver.http_client()
 
-    tenant_id, _ = env.neon_cli.create_tenant()
+    tenant_id = env.initial_tenant
 
     timelines_dir = env.pageserver.workdir / "tenants" / str(tenant_id) / "timelines"
     old_tenant_timelines = env.neon_cli.list_timelines(tenant_id)
diff --git a/test_runner/regress/test_ondemand_download.py b/test_runner/regress/test_ondemand_download.py
index 259c6412ef..fd9f9d631d 100644
--- a/test_runner/regress/test_ondemand_download.py
+++ b/test_runner/regress/test_ondemand_download.py
@@ -390,9 +390,19 @@ def test_download_remote_layers_api(
     wait_until(10, 0.2, lambda: assert_tenant_state(client, tenant_id, "Active"))
 
     ###### Phase 1: exercise download error code path
+
+    # comparison here is requiring the size to be at least the previous size, because it's possible received WAL after last_flush_lsn_upload
+    # witnessed for example difference of 29827072 (filled_current_physical) to 29868032 (here) is no good reason to fail a test.
+    this_time = get_api_current_physical_size()
     assert (
-        filled_current_physical == get_api_current_physical_size()
+        filled_current_physical <= this_time
     ), "current_physical_size is sum of loaded layer sizes, independent of whether local or remote"
+    if filled_current_physical != this_time:
+        log.info(
+            f"fixing up filled_current_physical from {filled_current_physical} to {this_time} ({this_time - filled_current_physical})"
+        )
+        filled_current_physical = this_time
+
     post_unlink_size = get_resident_physical_size()
     log.info(f"post_unlink_size: {post_unlink_size}")
     assert (
diff --git a/test_runner/regress/test_tenant_size.py b/test_runner/regress/test_tenant_size.py
index 25c6634108..49a6ca5a53 100644
--- a/test_runner/regress/test_tenant_size.py
+++ b/test_runner/regress/test_tenant_size.py
@@ -11,7 +11,10 @@ from fixtures.neon_fixtures import (
     wait_for_wal_insert_lsn,
 )
 from fixtures.pageserver.http import PageserverHttpClient
-from fixtures.pageserver.utils import timeline_delete_wait_completed
+from fixtures.pageserver.utils import (
+    timeline_delete_wait_completed,
+    wait_until_tenant_active,
+)
 from fixtures.pg_version import PgVersion, xfail_on_postgres
 from fixtures.types import Lsn, TenantId, TimelineId
 
@@ -517,6 +520,8 @@ def test_single_branch_get_tenant_size_grows(
     env.pageserver.stop()
     env.pageserver.start()
 
+    wait_until_tenant_active(http_client, tenant_id)
+
     size_after = http_client.tenant_size(tenant_id)
     size_debug = http_client.tenant_size_debug(tenant_id)
     size_debug_file.write(size_debug)
@@ -624,6 +629,8 @@ def test_get_tenant_size_with_multiple_branches(
     env.pageserver.stop()
     env.pageserver.start()
 
+    wait_until_tenant_active(http_client, tenant_id)
+
     # chance of compaction and gc on startup might have an effect on the
     # tenant_size but so far this has been reliable, even though at least gc
     # and tenant_size race for the same locks
diff --git a/test_runner/regress/test_timeline_delete.py b/test_runner/regress/test_timeline_delete.py
index 1e38e0534c..0b79383442 100644
--- a/test_runner/regress/test_timeline_delete.py
+++ b/test_runner/regress/test_timeline_delete.py
@@ -128,6 +128,8 @@ def test_timeline_delete(neon_simple_env: NeonEnv):
     env.pageserver.stop(immediate=True)
     env.pageserver.start()
 
+    wait_until_tenant_active(ps_http, env.initial_tenant)
+
     with pytest.raises(
         PageserverApiException,
         match=f"Timeline {env.initial_tenant}/{leaf_timeline_id} was not found",
diff --git a/test_runner/regress/test_timeline_size.py b/test_runner/regress/test_timeline_size.py
index ccdfc18de8..3f379a173c 100644
--- a/test_runner/regress/test_timeline_size.py
+++ b/test_runner/regress/test_timeline_size.py
@@ -367,10 +367,13 @@ def test_timeline_physical_size_post_checkpoint(
     wait_for_last_flush_lsn(env, endpoint, env.initial_tenant, new_timeline_id)
     pageserver_http.timeline_checkpoint(env.initial_tenant, new_timeline_id)
 
-    assert_physical_size_invariants(
-        get_physical_size_values(env, env.initial_tenant, new_timeline_id, remote_storage_kind),
-        remote_storage_kind,
-    )
+    def check():
+        assert_physical_size_invariants(
+            get_physical_size_values(env, env.initial_tenant, new_timeline_id, remote_storage_kind),
+            remote_storage_kind,
+        )
+
+    wait_until(10, 1, check)
 
 
 @pytest.mark.parametrize("remote_storage_kind", [None, RemoteStorageKind.LOCAL_FS])

From d7fa2dba2d3eaad9f7693f25ae07ed77f5ba9bf8 Mon Sep 17 00:00:00 2001
From: duguorong009 <80258679+duguorong009@users.noreply.github.com>
Date: Mon, 11 Sep 2023 19:58:15 +0800
Subject: [PATCH 56/61] fix(pageserver): update the `STORAGE_IO_TIME` metrics
 to avoid expensive operations (#5273)

Introduce the `StorageIoOperation` enum, `StorageIoTime` struct, and
`STORAGE_IO_TIME_METRIC` static which provides lockless access to
histograms consumed by `VirtualFile`.

Closes #5131

Co-authored-by: Joonas Koivunen <joonas@neon.tech>
---
 pageserver/src/metrics.rs      | 71 +++++++++++++++++++++++++++++-----
 pageserver/src/virtual_file.rs | 43 +++++++++++---------
 2 files changed, 85 insertions(+), 29 deletions(-)

diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs
index e024a9d77a..691ccafae1 100644
--- a/pageserver/src/metrics.rs
+++ b/pageserver/src/metrics.rs
@@ -537,7 +537,7 @@ const STORAGE_IO_TIME_BUCKETS: &[f64] = &[
     30.000,   // 30000 ms
 ];
 
-/// Tracks time taken by fs operations near VirtualFile.
+/// VirtualFile fs operation variants.
 ///
 /// Operations:
 /// - open ([`std::fs::OpenOptions::open`])
@@ -548,15 +548,66 @@ const STORAGE_IO_TIME_BUCKETS: &[f64] = &[
 /// - seek (modify internal position or file length query)
 /// - fsync ([`std::fs::File::sync_all`])
 /// - metadata ([`std::fs::File::metadata`])
-pub(crate) static STORAGE_IO_TIME: Lazy<HistogramVec> = Lazy::new(|| {
-    register_histogram_vec!(
-        "pageserver_io_operations_seconds",
-        "Time spent in IO operations",
-        &["operation"],
-        STORAGE_IO_TIME_BUCKETS.into()
-    )
-    .expect("failed to define a metric")
-});
+#[derive(
+    Debug, Clone, Copy, strum_macros::EnumCount, strum_macros::EnumIter, strum_macros::FromRepr,
+)]
+pub(crate) enum StorageIoOperation {
+    Open,
+    Close,
+    CloseByReplace,
+    Read,
+    Write,
+    Seek,
+    Fsync,
+    Metadata,
+}
+
+impl StorageIoOperation {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            StorageIoOperation::Open => "open",
+            StorageIoOperation::Close => "close",
+            StorageIoOperation::CloseByReplace => "close-by-replace",
+            StorageIoOperation::Read => "read",
+            StorageIoOperation::Write => "write",
+            StorageIoOperation::Seek => "seek",
+            StorageIoOperation::Fsync => "fsync",
+            StorageIoOperation::Metadata => "metadata",
+        }
+    }
+}
+
+/// Tracks time taken by fs operations near VirtualFile.
+#[derive(Debug)]
+pub(crate) struct StorageIoTime {
+    metrics: [Histogram; StorageIoOperation::COUNT],
+}
+
+impl StorageIoTime {
+    fn new() -> Self {
+        let storage_io_histogram_vec = register_histogram_vec!(
+            "pageserver_io_operations_seconds",
+            "Time spent in IO operations",
+            &["operation"],
+            STORAGE_IO_TIME_BUCKETS.into()
+        )
+        .expect("failed to define a metric");
+        let metrics = std::array::from_fn(|i| {
+            let op = StorageIoOperation::from_repr(i).unwrap();
+            let metric = storage_io_histogram_vec
+                .get_metric_with_label_values(&[op.as_str()])
+                .unwrap();
+            metric
+        });
+        Self { metrics }
+    }
+
+    pub(crate) fn get(&self, op: StorageIoOperation) -> &Histogram {
+        &self.metrics[op as usize]
+    }
+}
+
+pub(crate) static STORAGE_IO_TIME_METRIC: Lazy<StorageIoTime> = Lazy::new(StorageIoTime::new);
 
 const STORAGE_IO_SIZE_OPERATIONS: &[&str] = &["read", "write"];
 
diff --git a/pageserver/src/virtual_file.rs b/pageserver/src/virtual_file.rs
index 1fa5fcc297..9612b8ec6d 100644
--- a/pageserver/src/virtual_file.rs
+++ b/pageserver/src/virtual_file.rs
@@ -10,7 +10,7 @@
 //! This is similar to PostgreSQL's virtual file descriptor facility in
 //! src/backend/storage/file/fd.c
 //!
-use crate::metrics::{STORAGE_IO_SIZE, STORAGE_IO_TIME};
+use crate::metrics::{StorageIoOperation, STORAGE_IO_SIZE, STORAGE_IO_TIME_METRIC};
 use crate::tenant::TENANTS_SEGMENT_NAME;
 use once_cell::sync::OnceCell;
 use std::fs::{self, File, OpenOptions};
@@ -155,8 +155,8 @@ impl OpenFiles {
         if let Some(old_file) = slot_guard.file.take() {
             // the normal path of dropping VirtualFile uses "close", use "close-by-replace" here to
             // distinguish the two.
-            STORAGE_IO_TIME
-                .with_label_values(&["close-by-replace"])
+            STORAGE_IO_TIME_METRIC
+                .get(StorageIoOperation::CloseByReplace)
                 .observe_closure_duration(|| drop(old_file));
         }
 
@@ -245,8 +245,9 @@ impl VirtualFile {
             timeline_id = "*".to_string();
         }
         let (handle, mut slot_guard) = get_open_files().find_victim_slot();
-        let file = STORAGE_IO_TIME
-            .with_label_values(&["open"])
+
+        let file = STORAGE_IO_TIME_METRIC
+            .get(StorageIoOperation::Open)
             .observe_closure_duration(|| open_options.open(path))?;
 
         // Strip all options other than read and write.
@@ -330,17 +331,19 @@ impl VirtualFile {
 
     /// Call File::sync_all() on the underlying File.
     pub async fn sync_all(&self) -> Result<(), Error> {
-        self.with_file("fsync", |file| file.sync_all()).await?
+        self.with_file(StorageIoOperation::Fsync, |file| file.sync_all())
+            .await?
     }
 
     pub async fn metadata(&self) -> Result<fs::Metadata, Error> {
-        self.with_file("metadata", |file| file.metadata()).await?
+        self.with_file(StorageIoOperation::Metadata, |file| file.metadata())
+            .await?
     }
 
     /// Helper function that looks up the underlying File for this VirtualFile,
     /// opening it and evicting some other File if necessary. It calls 'func'
     /// with the physical File.
-    async fn with_file<F, R>(&self, op: &str, mut func: F) -> Result<R, Error>
+    async fn with_file<F, R>(&self, op: StorageIoOperation, mut func: F) -> Result<R, Error>
     where
         F: FnMut(&File) -> R,
     {
@@ -363,8 +366,8 @@ impl VirtualFile {
                         if let Some(file) = &slot_guard.file {
                             // Found a cached file descriptor.
                             slot.recently_used.store(true, Ordering::Relaxed);
-                            return Ok(STORAGE_IO_TIME
-                                .with_label_values(&[op])
+                            return Ok(STORAGE_IO_TIME_METRIC
+                                .get(op)
                                 .observe_closure_duration(|| func(file)));
                         }
                     }
@@ -390,13 +393,13 @@ impl VirtualFile {
         let (handle, mut slot_guard) = open_files.find_victim_slot();
 
         // Open the physical file
-        let file = STORAGE_IO_TIME
-            .with_label_values(&["open"])
+        let file = STORAGE_IO_TIME_METRIC
+            .get(StorageIoOperation::Open)
             .observe_closure_duration(|| self.open_options.open(&self.path))?;
 
         // Perform the requested operation on it
-        let result = STORAGE_IO_TIME
-            .with_label_values(&[op])
+        let result = STORAGE_IO_TIME_METRIC
+            .get(op)
             .observe_closure_duration(|| func(&file));
 
         // Store the File in the slot and update the handle in the VirtualFile
@@ -421,7 +424,9 @@ impl VirtualFile {
             }
             SeekFrom::End(offset) => {
                 self.pos = self
-                    .with_file("seek", |mut file| file.seek(SeekFrom::End(offset)))
+                    .with_file(StorageIoOperation::Seek, |mut file| {
+                        file.seek(SeekFrom::End(offset))
+                    })
                     .await??
             }
             SeekFrom::Current(offset) => {
@@ -511,7 +516,7 @@ impl VirtualFile {
 
     pub async fn read_at(&self, buf: &mut [u8], offset: u64) -> Result<usize, Error> {
         let result = self
-            .with_file("read", |file| file.read_at(buf, offset))
+            .with_file(StorageIoOperation::Read, |file| file.read_at(buf, offset))
             .await?;
         if let Ok(size) = result {
             STORAGE_IO_SIZE
@@ -523,7 +528,7 @@ impl VirtualFile {
 
     async fn write_at(&self, buf: &[u8], offset: u64) -> Result<usize, Error> {
         let result = self
-            .with_file("write", |file| file.write_at(buf, offset))
+            .with_file(StorageIoOperation::Write, |file| file.write_at(buf, offset))
             .await?;
         if let Ok(size) = result {
             STORAGE_IO_SIZE
@@ -576,8 +581,8 @@ impl Drop for VirtualFile {
             slot.recently_used.store(false, Ordering::Relaxed);
             // there is also operation "close-by-replace" for closes done on eviction for
             // comparison.
-            STORAGE_IO_TIME
-                .with_label_values(&["close"])
+            STORAGE_IO_TIME_METRIC
+                .get(StorageIoOperation::Close)
                 .observe_closure_duration(|| drop(slot_guard.file.take()));
         }
     }

From c0ed362790caa368aa65ba57d352a2f1562fd6bf Mon Sep 17 00:00:00 2001
From: bojanserafimov <bojan.serafimov7@gmail.com>
Date: Mon, 11 Sep 2023 09:46:06 -0400
Subject: [PATCH 57/61] Measure pageserver wal recovery time and fix flush()
 method (#5240)

---
 test_runner/fixtures/compare_fixtures.py    |  2 ++
 test_runner/fixtures/pageserver/utils.py    | 13 +++++----
 test_runner/performance/test_bulk_insert.py | 31 ++++++++++++++++++++-
 3 files changed, 39 insertions(+), 7 deletions(-)

diff --git a/test_runner/fixtures/compare_fixtures.py b/test_runner/fixtures/compare_fixtures.py
index a10ef70aa2..1254c4e779 100644
--- a/test_runner/fixtures/compare_fixtures.py
+++ b/test_runner/fixtures/compare_fixtures.py
@@ -14,6 +14,7 @@ from fixtures.neon_fixtures import (
     PgProtocol,
     RemotePostgres,
     VanillaPostgres,
+    wait_for_last_flush_lsn,
 )
 from fixtures.pg_stats import PgStatTable
 
@@ -129,6 +130,7 @@ class NeonCompare(PgCompare):
         return self._pg_bin
 
     def flush(self):
+        wait_for_last_flush_lsn(self.env, self._pg, self.tenant, self.timeline)
         self.pageserver_http_client.timeline_checkpoint(self.tenant, self.timeline)
         self.pageserver_http_client.timeline_gc(self.tenant, self.timeline, 0)
 
diff --git a/test_runner/fixtures/pageserver/utils.py b/test_runner/fixtures/pageserver/utils.py
index 7d6dec4a14..3197017bd1 100644
--- a/test_runner/fixtures/pageserver/utils.py
+++ b/test_runner/fixtures/pageserver/utils.py
@@ -154,16 +154,17 @@ def wait_for_last_record_lsn(
     lsn: Lsn,
 ) -> Lsn:
     """waits for pageserver to catch up to a certain lsn, returns the last observed lsn."""
-    for i in range(10):
+    for i in range(100):
         current_lsn = last_record_lsn(pageserver_http, tenant, timeline)
         if current_lsn >= lsn:
             return current_lsn
-        log.info(
-            "waiting for last_record_lsn to reach {}, now {}, iteration {}".format(
-                lsn, current_lsn, i + 1
+        if i % 10 == 0:
+            log.info(
+                "waiting for last_record_lsn to reach {}, now {}, iteration {}".format(
+                    lsn, current_lsn, i + 1
+                )
             )
-        )
-        time.sleep(1)
+        time.sleep(0.1)
     raise Exception(
         "timed out while waiting for last_record_lsn to reach {}, was {}".format(lsn, current_lsn)
     )
diff --git a/test_runner/performance/test_bulk_insert.py b/test_runner/performance/test_bulk_insert.py
index d6e67aa361..92da2dae96 100644
--- a/test_runner/performance/test_bulk_insert.py
+++ b/test_runner/performance/test_bulk_insert.py
@@ -1,6 +1,8 @@
+import shutil
 from contextlib import closing
 
-from fixtures.compare_fixtures import PgCompare
+from fixtures.compare_fixtures import NeonCompare, PgCompare
+from fixtures.pg_version import PgVersion
 
 
 #
@@ -28,3 +30,30 @@ def test_bulk_insert(neon_with_baseline: PgCompare):
 
             env.report_peak_memory_use()
             env.report_size()
+
+    # When testing neon, also check how long it takes the pageserver to reingest the
+    # wal from safekeepers. If this number is close to total runtime, then the pageserver
+    # is the bottleneck.
+    if isinstance(env, NeonCompare):
+        measure_recovery_time(env)
+
+
+def measure_recovery_time(env: NeonCompare):
+    client = env.env.pageserver.http_client()
+    pg_version = PgVersion(client.timeline_detail(env.tenant, env.timeline)["pg_version"])
+
+    # Stop pageserver and remove tenant data
+    env.env.pageserver.stop()
+    timeline_dir = env.env.timeline_dir(env.tenant, env.timeline)
+    shutil.rmtree(timeline_dir)
+
+    # Start pageserver
+    env.env.pageserver.start()
+
+    # Measure recovery time
+    with env.record_duration("wal_recovery"):
+        # Create the tenant, which will start walingest
+        client.timeline_create(pg_version, env.tenant, env.timeline)
+
+        # Flush, which will also wait for lsn to catch up
+        env.flush()

From 76cc87398c58aa8856083bd3b17403af56715b17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arpad=20M=C3=BCller?= <arpad-m@users.noreply.github.com>
Date: Mon, 11 Sep 2023 17:35:05 +0200
Subject: [PATCH 58/61] Use tokio locks in VirtualFile and turn with_file into
 macro (#5247)

## Problem

For #4743, we want to convert everything up to the actual I/O operations
of `VirtualFile` to `async fn`.

## Summary of changes

This PR is the last change in a series of changes to `VirtualFile`:
#5189, #5190, #5195, #5203, and #5224.

It does the last preparations before the I/O operations are actually
made async. We are doing the following things:

* First, we change the locks for the file descriptor cache to tokio's
locks that support Send. This is important when one wants to hold locks
across await points (which we want to do), otherwise the Future won't be
Send. Also, one shouldn't generally block in async code as executors
don't like that.
* Due to the lock change, we now take an approach for the `VirtualFile`
destructors similar to the one proposed by #5122 for the page cache, to
use `try_write`. Similarly to the situation in the linked PR, one can
make an argument that if we are in the destructor and the slot has not
been reused yet, we are the only user accessing the slot due to owning
the lock mutably. It is still possible that we are not obtaining the
lock, but the only cause for that is the clock algorithm touching the
slot, which should be quite an unlikely occurence. For the instance of
`try_write` failing, we spawn an async task to destroy the lock. As just
argued however, most of the time the code path where we spawn the task
should not be visited.
* Lastly, we split `with_file` into a macro part, and a function part
that contains most of the logic. The function part returns a lock
object, that the macro uses. The macro exists to perform the operation
in a more compact fashion, saving code from putting the lock into a
variable and then doing the operation while measuring the time to run
it. We take the locks approach because Rust has no support for async
closures. One can make normal closures return a future, but that
approach gets into lifetime issues the moment you want to pass data to
these closures via parameters that has a lifetime (captures work). For
details, see
[this](https://smallcultfollowing.com/babysteps/blog/2023/03/29/thoughts-on-async-closures/)
and
[this](https://users.rust-lang.org/t/function-that-takes-an-async-closure/61663)
link. In #5224, we ran into a similar problem with the `test_files`
function, and we ended up passing the path and the `OpenOptions`
by-value instead of by-ref, at the expense of a few extra copies. This
can be done as the data is cheaply copyable, and we are in test code.
But here, we are not, and while `File::try_clone` exists, it [issues
system calls
internally](https://github.com/rust-lang/rust/blob/1e746d7741d44551e9378daf13b8797322aa0b74/library/std/src/os/fd/owned.rs#L94-L111).
Also, it would allocate an entirely new file descriptor, something that
the fd cache was built to prevent.
* We change the `STORAGE_IO_TIME` metrics to support async.

Part of #4743.
---
 pageserver/src/virtual_file.rs | 150 +++++++++++++++++++++------------
 1 file changed, 94 insertions(+), 56 deletions(-)

diff --git a/pageserver/src/virtual_file.rs b/pageserver/src/virtual_file.rs
index 9612b8ec6d..5dc7e4e490 100644
--- a/pageserver/src/virtual_file.rs
+++ b/pageserver/src/virtual_file.rs
@@ -18,7 +18,8 @@ use std::io::{Error, ErrorKind, Seek, SeekFrom};
 use std::os::unix::fs::FileExt;
 use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
-use std::sync::{RwLock, RwLockWriteGuard};
+use tokio::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard};
+use tokio::time::Instant;
 
 ///
 /// A virtual file descriptor. You can use this just like std::fs::File, but internally
@@ -110,7 +111,7 @@ impl OpenFiles {
     ///
     /// On return, we hold a lock on the slot, and its 'tag' has been updated
     /// recently_used has been set. It's all ready for reuse.
-    fn find_victim_slot(&self) -> (SlotHandle, RwLockWriteGuard<SlotInner>) {
+    async fn find_victim_slot(&self) -> (SlotHandle, RwLockWriteGuard<SlotInner>) {
         //
         // Run the clock algorithm to find a slot to replace.
         //
@@ -142,7 +143,7 @@ impl OpenFiles {
                 }
                 retries += 1;
             } else {
-                slot_guard = slot.inner.write().unwrap();
+                slot_guard = slot.inner.write().await;
                 index = next;
                 break;
             }
@@ -208,6 +209,19 @@ impl CrashsafeOverwriteError {
     }
 }
 
+macro_rules! with_file {
+    ($this:expr, $op:expr, | $ident:ident | $($body:tt)*) => {{
+        let $ident = $this.lock_file().await?;
+        let instant = Instant::now();
+        let result = $($body)*;
+        let elapsed = instant.elapsed().as_secs_f64();
+        STORAGE_IO_TIME_METRIC
+            .get($op)
+            .observe(elapsed);
+        result
+    }};
+}
+
 impl VirtualFile {
     /// Open a file in read-only mode. Like File::open.
     pub async fn open(path: &Path) -> Result<VirtualFile, std::io::Error> {
@@ -244,7 +258,7 @@ impl VirtualFile {
             tenant_id = "*".to_string();
             timeline_id = "*".to_string();
         }
-        let (handle, mut slot_guard) = get_open_files().find_victim_slot();
+        let (handle, mut slot_guard) = get_open_files().find_victim_slot().await;
 
         let file = STORAGE_IO_TIME_METRIC
             .get(StorageIoOperation::Open)
@@ -331,22 +345,24 @@ impl VirtualFile {
 
     /// Call File::sync_all() on the underlying File.
     pub async fn sync_all(&self) -> Result<(), Error> {
-        self.with_file(StorageIoOperation::Fsync, |file| file.sync_all())
-            .await?
+        with_file!(self, StorageIoOperation::Fsync, |file| file
+            .as_ref()
+            .sync_all())
     }
 
     pub async fn metadata(&self) -> Result<fs::Metadata, Error> {
-        self.with_file(StorageIoOperation::Metadata, |file| file.metadata())
-            .await?
+        with_file!(self, StorageIoOperation::Metadata, |file| file
+            .as_ref()
+            .metadata())
     }
 
-    /// Helper function that looks up the underlying File for this VirtualFile,
-    /// opening it and evicting some other File if necessary. It calls 'func'
-    /// with the physical File.
-    async fn with_file<F, R>(&self, op: StorageIoOperation, mut func: F) -> Result<R, Error>
-    where
-        F: FnMut(&File) -> R,
-    {
+    /// Helper function internal to `VirtualFile` that looks up the underlying File,
+    /// opens it and evicts some other File if necessary. The passed parameter is
+    /// assumed to be a function available for the physical `File`.
+    ///
+    /// We are doing it via a macro as Rust doesn't support async closures that
+    /// take on parameters with lifetimes.
+    async fn lock_file(&self) -> Result<FileGuard<'_>, Error> {
         let open_files = get_open_files();
 
         let mut handle_guard = {
@@ -356,27 +372,23 @@ impl VirtualFile {
             // We only need to hold the handle lock while we read the current handle. If
             // another thread closes the file and recycles the slot for a different file,
             // we will notice that the handle we read is no longer valid and retry.
-            let mut handle = *self.handle.read().unwrap();
+            let mut handle = *self.handle.read().await;
             loop {
                 // Check if the slot contains our File
                 {
                     let slot = &open_files.slots[handle.index];
-                    let slot_guard = slot.inner.read().unwrap();
-                    if slot_guard.tag == handle.tag {
-                        if let Some(file) = &slot_guard.file {
-                            // Found a cached file descriptor.
-                            slot.recently_used.store(true, Ordering::Relaxed);
-                            return Ok(STORAGE_IO_TIME_METRIC
-                                .get(op)
-                                .observe_closure_duration(|| func(file)));
-                        }
+                    let slot_guard = slot.inner.read().await;
+                    if slot_guard.tag == handle.tag && slot_guard.file.is_some() {
+                        // Found a cached file descriptor.
+                        slot.recently_used.store(true, Ordering::Relaxed);
+                        return Ok(FileGuard { slot_guard });
                     }
                 }
 
                 // The slot didn't contain our File. We will have to open it ourselves,
                 // but before that, grab a write lock on handle in the VirtualFile, so
                 // that no other thread will try to concurrently open the same file.
-                let handle_guard = self.handle.write().unwrap();
+                let handle_guard = self.handle.write().await;
 
                 // If another thread changed the handle while we were not holding the lock,
                 // then the handle might now be valid again. Loop back to retry.
@@ -390,25 +402,22 @@ impl VirtualFile {
 
         // We need to open the file ourselves. The handle in the VirtualFile is
         // now locked in write-mode. Find a free slot to put it in.
-        let (handle, mut slot_guard) = open_files.find_victim_slot();
+        let (handle, mut slot_guard) = open_files.find_victim_slot().await;
 
         // Open the physical file
         let file = STORAGE_IO_TIME_METRIC
             .get(StorageIoOperation::Open)
             .observe_closure_duration(|| self.open_options.open(&self.path))?;
 
-        // Perform the requested operation on it
-        let result = STORAGE_IO_TIME_METRIC
-            .get(op)
-            .observe_closure_duration(|| func(&file));
-
         // Store the File in the slot and update the handle in the VirtualFile
         // to point to it.
         slot_guard.file.replace(file);
 
         *handle_guard = handle;
 
-        Ok(result)
+        return Ok(FileGuard {
+            slot_guard: slot_guard.downgrade(),
+        });
     }
 
     pub fn remove(self) {
@@ -423,11 +432,9 @@ impl VirtualFile {
                 self.pos = offset;
             }
             SeekFrom::End(offset) => {
-                self.pos = self
-                    .with_file(StorageIoOperation::Seek, |mut file| {
-                        file.seek(SeekFrom::End(offset))
-                    })
-                    .await??
+                self.pos = with_file!(self, StorageIoOperation::Seek, |file| file
+                    .as_ref()
+                    .seek(SeekFrom::End(offset)))?
             }
             SeekFrom::Current(offset) => {
                 let pos = self.pos as i128 + offset as i128;
@@ -515,9 +522,9 @@ impl VirtualFile {
     }
 
     pub async fn read_at(&self, buf: &mut [u8], offset: u64) -> Result<usize, Error> {
-        let result = self
-            .with_file(StorageIoOperation::Read, |file| file.read_at(buf, offset))
-            .await?;
+        let result = with_file!(self, StorageIoOperation::Read, |file| file
+            .as_ref()
+            .read_at(buf, offset));
         if let Ok(size) = result {
             STORAGE_IO_SIZE
                 .with_label_values(&["read", &self.tenant_id, &self.timeline_id])
@@ -527,9 +534,9 @@ impl VirtualFile {
     }
 
     async fn write_at(&self, buf: &[u8], offset: u64) -> Result<usize, Error> {
-        let result = self
-            .with_file(StorageIoOperation::Write, |file| file.write_at(buf, offset))
-            .await?;
+        let result = with_file!(self, StorageIoOperation::Write, |file| file
+            .as_ref()
+            .write_at(buf, offset));
         if let Ok(size) = result {
             STORAGE_IO_SIZE
                 .with_label_values(&["write", &self.tenant_id, &self.timeline_id])
@@ -539,6 +546,18 @@ impl VirtualFile {
     }
 }
 
+struct FileGuard<'a> {
+    slot_guard: RwLockReadGuard<'a, SlotInner>,
+}
+
+impl<'a> AsRef<File> for FileGuard<'a> {
+    fn as_ref(&self) -> &File {
+        // This unwrap is safe because we only create `FileGuard`s
+        // if we know that the file is Some.
+        self.slot_guard.file.as_ref().unwrap()
+    }
+}
+
 #[cfg(test)]
 impl VirtualFile {
     pub(crate) async fn read_blk(
@@ -571,20 +590,39 @@ impl VirtualFile {
 impl Drop for VirtualFile {
     /// If a VirtualFile is dropped, close the underlying file if it was open.
     fn drop(&mut self) {
-        let handle = self.handle.get_mut().unwrap();
+        let handle = self.handle.get_mut();
 
-        // We could check with a read-lock first, to avoid waiting on an
-        // unrelated I/O.
-        let slot = &get_open_files().slots[handle.index];
-        let mut slot_guard = slot.inner.write().unwrap();
-        if slot_guard.tag == handle.tag {
-            slot.recently_used.store(false, Ordering::Relaxed);
-            // there is also operation "close-by-replace" for closes done on eviction for
-            // comparison.
-            STORAGE_IO_TIME_METRIC
-                .get(StorageIoOperation::Close)
-                .observe_closure_duration(|| drop(slot_guard.file.take()));
+        fn clean_slot(slot: &Slot, mut slot_guard: RwLockWriteGuard<'_, SlotInner>, tag: u64) {
+            if slot_guard.tag == tag {
+                slot.recently_used.store(false, Ordering::Relaxed);
+                // there is also operation "close-by-replace" for closes done on eviction for
+                // comparison.
+                STORAGE_IO_TIME_METRIC
+                    .get(StorageIoOperation::Close)
+                    .observe_closure_duration(|| drop(slot_guard.file.take()));
+            }
         }
+
+        // We don't have async drop so we cannot directly await the lock here.
+        // Instead, first do a best-effort attempt at closing the underlying
+        // file descriptor by using `try_write`, and if that fails, spawn
+        // a tokio task to do it asynchronously: we just want it to be
+        // cleaned up eventually.
+        // Most of the time, the `try_lock` should succeed though,
+        // as we have `&mut self` access. In other words, if the slot
+        // is still occupied by our file, there should be no access from
+        // other I/O operations; the only other possible place to lock
+        // the slot is the lock algorithm looking for free slots.
+        let slot = &get_open_files().slots[handle.index];
+        if let Ok(slot_guard) = slot.inner.try_write() {
+            clean_slot(slot, slot_guard, handle.tag);
+        } else {
+            let tag = handle.tag;
+            tokio::spawn(async move {
+                let slot_guard = slot.inner.write().await;
+                clean_slot(slot, slot_guard, tag);
+            });
+        };
     }
 }
 

From a18d6d9ae3e1f395042eed39f06685dcaeb2ecc6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arpad=20M=C3=BCller?= <arpad-m@users.noreply.github.com>
Date: Mon, 11 Sep 2023 18:41:08 +0200
Subject: [PATCH 59/61] Make File opening in VirtualFile async-compatible
 (#5280)

## Problem

Previously, we were using `observe_closure_duration` in `VirtualFile`
file opening code, but this doesn't support async open operations, which
we want to use as part of #4743.

## Summary of changes

* Move the duration measurement from the `with_file` macro into a
`observe_duration` macro.
* Some smaller drive-by fixes to replace the old strings with the new
variant names introduced by #5273

Part of #4743, follow-up of #5247.
---
 pageserver/src/virtual_file.rs | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/pageserver/src/virtual_file.rs b/pageserver/src/virtual_file.rs
index 5dc7e4e490..e437ffb2d7 100644
--- a/pageserver/src/virtual_file.rs
+++ b/pageserver/src/virtual_file.rs
@@ -154,7 +154,7 @@ impl OpenFiles {
         // old file.
         //
         if let Some(old_file) = slot_guard.file.take() {
-            // the normal path of dropping VirtualFile uses "close", use "close-by-replace" here to
+            // the normal path of dropping VirtualFile uses `Close`, use `CloseByReplace` here to
             // distinguish the two.
             STORAGE_IO_TIME_METRIC
                 .get(StorageIoOperation::CloseByReplace)
@@ -209,9 +209,12 @@ impl CrashsafeOverwriteError {
     }
 }
 
-macro_rules! with_file {
-    ($this:expr, $op:expr, | $ident:ident | $($body:tt)*) => {{
-        let $ident = $this.lock_file().await?;
+/// Observe duration for the given storage I/O operation
+///
+/// Unlike `observe_closure_duration`, this supports async,
+/// where "support" means that we measure wall clock time.
+macro_rules! observe_duration {
+    ($op:expr, $($body:tt)*) => {{
         let instant = Instant::now();
         let result = $($body)*;
         let elapsed = instant.elapsed().as_secs_f64();
@@ -219,6 +222,13 @@ macro_rules! with_file {
             .get($op)
             .observe(elapsed);
         result
+    }}
+}
+
+macro_rules! with_file {
+    ($this:expr, $op:expr, | $ident:ident | $($body:tt)*) => {{
+        let $ident = $this.lock_file().await?;
+        observe_duration!($op, $($body)*)
     }};
 }
 
@@ -260,9 +270,7 @@ impl VirtualFile {
         }
         let (handle, mut slot_guard) = get_open_files().find_victim_slot().await;
 
-        let file = STORAGE_IO_TIME_METRIC
-            .get(StorageIoOperation::Open)
-            .observe_closure_duration(|| open_options.open(path))?;
+        let file = observe_duration!(StorageIoOperation::Open, open_options.open(path))?;
 
         // Strip all options other than read and write.
         //
@@ -405,9 +413,7 @@ impl VirtualFile {
         let (handle, mut slot_guard) = open_files.find_victim_slot().await;
 
         // Open the physical file
-        let file = STORAGE_IO_TIME_METRIC
-            .get(StorageIoOperation::Open)
-            .observe_closure_duration(|| self.open_options.open(&self.path))?;
+        let file = observe_duration!(StorageIoOperation::Open, self.open_options.open(&self.path))?;
 
         // Store the File in the slot and update the handle in the VirtualFile
         // to point to it.
@@ -595,7 +601,7 @@ impl Drop for VirtualFile {
         fn clean_slot(slot: &Slot, mut slot_guard: RwLockWriteGuard<'_, SlotInner>, tag: u64) {
             if slot_guard.tag == tag {
                 slot.recently_used.store(false, Ordering::Relaxed);
-                // there is also operation "close-by-replace" for closes done on eviction for
+                // there is also the `CloseByReplace` operation for closes done on eviction for
                 // comparison.
                 STORAGE_IO_TIME_METRIC
                     .get(StorageIoOperation::Close)

From 15eaf78083ecff62b7669091da1a1c8b4f60ebf8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arpad=20M=C3=BCller?= <arpad-m@users.noreply.github.com>
Date: Tue, 12 Sep 2023 02:11:16 +0200
Subject: [PATCH 60/61] Disallow block_in_place and Handle::block_on (#5101)

## Problem

`block_in_place` is a quite expensive operation, and if it is used, we
should explicitly have to opt into it by allowing the
`clippy::disallowed_methods` lint.

For more, see
https://github.com/neondatabase/neon/pull/5023#discussion_r1304194495.

Similar arguments exist for `Handle::block_on`, but we don't do this yet
as there is still usages.

## Summary of changes

Adds a clippy.toml file, configuring the [`disallowed_methods`
lint](https://rust-lang.github.io/rust-clippy/master/#/disallowed_method).
---
 clippy.toml | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 clippy.toml

diff --git a/clippy.toml b/clippy.toml
new file mode 100644
index 0000000000..d788afc84d
--- /dev/null
+++ b/clippy.toml
@@ -0,0 +1,5 @@
+disallowed-methods = [
+    "tokio::task::block_in_place",
+    # Allow this for now, to deny it later once we stop using Handle::block_on completely
+    # "tokio::runtime::Handle::block_on",
+]

From 521438a5c6bbc5ecc061f35593b2dbb1715e0e06 Mon Sep 17 00:00:00 2001
From: Christian Schwarz <christian@neon.tech>
Date: Tue, 12 Sep 2023 11:23:46 +0200
Subject: [PATCH 61/61] fix deadlock around TENANTS (#5285)

The sequence that can lead to a deadlock:

1. DELETE request gets all the way to `tenant.shutdown(progress,
false).await.is_err() ` , while holding TENANTS.read()
2. POST request for tenant creation comes in, calls `tenant_map_insert`,
it does `let mut guard = TENANTS.write().await;`
3. Something that `tenant.shutdown()` needs to wait for needs a
`TENANTS.read().await`.
The only case identified in exhaustive manual scanning of the code base
is this one:
Imitate size access does `get_tenant().await`, which does
`TENANTS.read().await` under the hood.

In the above case (1) waits for (3), (3)'s read-lock request is queued
behind (2)'s write-lock, and (2) waits for (1).
Deadlock.

I made a reproducer/proof-that-above-hypothesis-holds in
https://github.com/neondatabase/neon/pull/5281 , but, it's not ready for
merge yet and we want the fix _now_.

fixes https://github.com/neondatabase/neon/issues/5284
---
 pageserver/src/tenant/mgr.rs                  |  2 ++
 .../src/tenant/timeline/eviction_task.rs      | 21 ++++++++++++++++---
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs
index 0cf5c36b87..74faee1115 100644
--- a/pageserver/src/tenant/mgr.rs
+++ b/pageserver/src/tenant/mgr.rs
@@ -503,6 +503,8 @@ pub enum GetTenantError {
 
 /// Gets the tenant from the in-memory data, erroring if it's absent or is not fitting to the query.
 /// `active_only = true` allows to query only tenants that are ready for operations, erroring on other kinds of tenants.
+///
+/// This method is cancel-safe.
 pub async fn get_tenant(
     tenant_id: TenantId,
     active_only: bool,
diff --git a/pageserver/src/tenant/timeline/eviction_task.rs b/pageserver/src/tenant/timeline/eviction_task.rs
index 3e407dda57..39f0d03a01 100644
--- a/pageserver/src/tenant/timeline/eviction_task.rs
+++ b/pageserver/src/tenant/timeline/eviction_task.rs
@@ -328,9 +328,24 @@ impl Timeline {
         // Make one of the tenant's timelines draw the short straw and run the calculation.
         // The others wait until the calculation is done so that they take into account the
         // imitated accesses that the winner made.
-        let Ok(tenant) = crate::tenant::mgr::get_tenant(self.tenant_id, true).await else {
-            // likely, we're shutting down
-            return ControlFlow::Break(());
+        //
+        // It is critical we are responsive to cancellation here. Otherwise, we deadlock with
+        // tenant deletion (holds TENANTS in read mode) any other task that attempts to
+        // acquire TENANTS in write mode before we here call get_tenant.
+        // See https://github.com/neondatabase/neon/issues/5284.
+        let res = tokio::select! {
+            _ = cancel.cancelled() => {
+                return ControlFlow::Break(());
+            }
+            res = crate::tenant::mgr::get_tenant(self.tenant_id, true) => {
+                res
+            }
+        };
+        let tenant = match res {
+            Ok(t) => t,
+            Err(_) => {
+                return ControlFlow::Break(());
+            }
         };
         let mut state = tenant.eviction_task_tenant_state.lock().await;
         match state.last_layer_access_imitation {