From 37fdbc3aaa85e76ec146b12478fbc3a259cebf4b Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 7 Dec 2023 21:36:44 +0200 Subject: [PATCH] fix: use larger buffers for remote storage (#6069) Currently using 8kB buffers, raise that to 32kB to hopefully 1/4 of `spawn_blocking` usage. Also a drive-by fixing of last `tokio::io::copy` to `tokio::io::copy_buf`. --- libs/remote_storage/src/local_fs.rs | 2 +- pageserver/src/tenant/remote_timeline_client.rs | 3 +++ pageserver/src/tenant/remote_timeline_client/download.rs | 2 +- pageserver/src/tenant/remote_timeline_client/upload.rs | 2 +- safekeeper/src/wal_backup.rs | 7 +++++-- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/libs/remote_storage/src/local_fs.rs b/libs/remote_storage/src/local_fs.rs index 0016c21955..03b98e5ea2 100644 --- a/libs/remote_storage/src/local_fs.rs +++ b/libs/remote_storage/src/local_fs.rs @@ -260,7 +260,7 @@ impl RemoteStorage for LocalFs { let mut buffer_to_read = data.take(from_size_bytes); // alternatively we could just write the bytes to a file, but local_fs is a testing utility - let bytes_read = io::copy(&mut buffer_to_read, &mut destination) + let bytes_read = io::copy_buf(&mut buffer_to_read, &mut destination) .await .with_context(|| { format!( diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 5b649a420c..1ef9fe4a64 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -254,6 +254,9 @@ pub(crate) const FAILED_UPLOAD_WARN_THRESHOLD: u32 = 3; pub(crate) const INITDB_PATH: &str = "initdb.tar.zst"; +/// Default buffer size when interfacing with [`tokio::fs::File`]. +const BUFFER_SIZE: usize = 32 * 1024; + pub enum MaybeDeletedIndexPart { IndexPart(IndexPart), Deleted(IndexPart), diff --git a/pageserver/src/tenant/remote_timeline_client/download.rs b/pageserver/src/tenant/remote_timeline_client/download.rs index 3356f55f34..ce942b56f8 100644 --- a/pageserver/src/tenant/remote_timeline_client/download.rs +++ b/pageserver/src/tenant/remote_timeline_client/download.rs @@ -90,7 +90,7 @@ pub async fn download_layer_file<'a>( .map_err(DownloadError::Other)?; let mut destination_file = - tokio::io::BufWriter::with_capacity(8 * 1024, destination_file); + tokio::io::BufWriter::with_capacity(super::BUFFER_SIZE, destination_file); let mut reader = tokio_util::io::StreamReader::new(download.download_stream); diff --git a/pageserver/src/tenant/remote_timeline_client/upload.rs b/pageserver/src/tenant/remote_timeline_client/upload.rs index 0ec539a64e..e1dea3ab4b 100644 --- a/pageserver/src/tenant/remote_timeline_client/upload.rs +++ b/pageserver/src/tenant/remote_timeline_client/upload.rs @@ -105,7 +105,7 @@ pub(super) async fn upload_timeline_layer<'a>( let fs_size = usize::try_from(fs_size) .with_context(|| format!("convert {source_path:?} size {fs_size} usize"))?; - let reader = tokio_util::io::ReaderStream::with_capacity(source_file, 8 * 1024); + let reader = tokio_util::io::ReaderStream::with_capacity(source_file, super::BUFFER_SIZE); storage .upload(reader, fs_size, &storage_path, None) diff --git a/safekeeper/src/wal_backup.rs b/safekeeper/src/wal_backup.rs index 2e2cb11e3f..c99bbc7d61 100644 --- a/safekeeper/src/wal_backup.rs +++ b/safekeeper/src/wal_backup.rs @@ -35,6 +35,9 @@ use once_cell::sync::OnceCell; const UPLOAD_FAILURE_RETRY_MIN_MS: u64 = 10; const UPLOAD_FAILURE_RETRY_MAX_MS: u64 = 5000; +/// Default buffer size when interfacing with [`tokio::fs::File`]. +const BUFFER_SIZE: usize = 32 * 1024; + /// Check whether wal backup is required for timeline. If yes, mark that launcher is /// aware of current status and return the timeline. async fn is_wal_backup_required(ttid: TenantTimelineId) -> Option> { @@ -498,7 +501,7 @@ async fn backup_object( .await .with_context(|| format!("Failed to open file {source_file:?} for wal backup"))?; - let file = tokio_util::io::ReaderStream::with_capacity(file, 8 * 1024); + let file = tokio_util::io::ReaderStream::with_capacity(file, BUFFER_SIZE); storage.upload_storage_object(file, size, target_file).await } @@ -524,7 +527,7 @@ pub async fn read_object( let reader = tokio_util::io::StreamReader::new(download.download_stream); - let reader = tokio::io::BufReader::with_capacity(8 * 1024, reader); + let reader = tokio::io::BufReader::with_capacity(BUFFER_SIZE, reader); Ok(Box::pin(reader)) }