avoid Vec::new() in walredo code path; still no dramatic improvement over before_scratch.svg

pageserver/src/tenant/blob_io.rs

@@ -25,6 +25,7 @@ impl<'a> BlockCursor<'a> {
         offset: u64,
         ctx: &RequestContext,
     ) -> Result<Vec<u8>, std::io::Error> {
+        // TODO: used pooled allocation instead, used by ImageLayer::get_value_reconstruct_data
         let mut buf = Vec::new();
         self.read_blob_into_buf(offset, &mut buf, ctx).await?;
         Ok(buf)

pageserver/src/walredo.rs

@@ -112,6 +112,8 @@ fn can_apply_in_neon(rec: &NeonWalRecord) -> bool {
     }
 }
 
+mod writebuf_pool;
+
 ///
 /// Public interface of WAL redo manager
 ///
@@ -798,7 +800,8 @@ impl WalRedoProcess {
         // Most requests start with a before-image with BLCKSZ bytes, followed by
         // by some other WAL records. Start with a buffer that can hold that
         // comfortably.
-        let mut writebuf: Vec<u8> = Vec::with_capacity((BLCKSZ as usize) * 3);
+        // TODO replace with allocation pool
+        let mut writebuf: writebuf_pool::PooledVecU8 = writebuf_pool::get();
         build_begin_redo_for_block_msg(tag, &mut writebuf);
         if let Some(img) = base_img {
             build_push_page_msg(tag, img, &mut writebuf);

pageserver/src/walredo/writebuf_pool.rs

@@ -0,0 +1,40 @@
+use std::cell::RefCell;
+
+use postgres_ffi::BLCKSZ;
+
+pub struct PooledVecU8(Option<Vec<u8>>);
+
+// Thread-local list of re-usable buffers.
+thread_local! {
+    static POOL: RefCell<Vec<Vec<u8>>> = RefCell::new(Vec::new());
+}
+
+pub(crate) fn get() -> PooledVecU8 {
+    let maybe = POOL.with(|rc| rc.borrow_mut().pop());
+    match maybe {
+        Some(buf) => PooledVecU8(Some(buf)),
+        None => PooledVecU8(Some(Vec::with_capacity((BLCKSZ as usize) * 3))),
+    }
+}
+
+impl Drop for PooledVecU8 {
+    fn drop(&mut self) {
+        let mut buf = self.0.take().unwrap();
+        buf.clear();
+        POOL.with(|rc| rc.borrow_mut().push(buf))
+    }
+}
+
+impl std::ops::Deref for PooledVecU8 {
+    type Target = Vec<u8>;
+
+    fn deref(&self) -> &Self::Target {
+        self.0.as_ref().unwrap()
+    }
+}
+
+impl std::ops::DerefMut for PooledVecU8 {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.0.as_mut().unwrap()
+    }
+}