rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-05-30 19:40:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: use larger buffers for remote storage (#6069)

Browse Source 
Currently using 8kB buffers, raise that to 32kB to hopefully 1/4 of
`spawn_blocking` usage. Also a drive-by fixing of last `tokio::io::copy`
to `tokio::io::copy_buf`.
This commit is contained in:
Joonas Koivunen
2023-12-07 21:36:44 +02:00
committed by GitHub
parent 7aa1e58301
commit 37fdbc3aaa
5 changed files with 11 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
safekeeper/src/wal_backup.rs
View File

@@ -35,6 +35,9 @@ use once_cell::sync::OnceCell;
const UPLOAD_FAILURE_RETRY_MIN_MS: u64 = 10;
const UPLOAD_FAILURE_RETRY_MAX_MS: u64 = 5000;
/// Default buffer size when interfacing with [`tokio::fs::File`].
const BUFFER_SIZE: usize = 32 * 1024;
/// Check whether wal backup is required for timeline. If yes, mark that launcher is
/// aware of current status and return the timeline.
async fn is_wal_backup_required(ttid: TenantTimelineId) -> Option<Arc<Timeline>> {
@@ -498,7 +501,7 @@ async fn backup_object(
.await
.with_context(|| format!("Failed to open file {source_file:?} for wal backup"))?;
let file = tokio_util::io::ReaderStream::with_capacity(file, 8 * 1024);
let file = tokio_util::io::ReaderStream::with_capacity(file, BUFFER_SIZE);
storage.upload_storage_object(file, size, target_file).await
}
@@ -524,7 +527,7 @@ pub async fn read_object(
let reader = tokio_util::io::StreamReader::new(download.download_stream);
let reader = tokio::io::BufReader::with_capacity(8 * 1024, reader);
let reader = tokio::io::BufReader::with_capacity(BUFFER_SIZE, reader);
Ok(Box::pin(reader))
}