rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-13 00:12:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: use larger buffers for remote storage (#6069)

Browse Source 
Currently using 8kB buffers, raise that to 32kB to hopefully 1/4 of
`spawn_blocking` usage. Also a drive-by fixing of last `tokio::io::copy`
to `tokio::io::copy_buf`.
This commit is contained in:
Joonas Koivunen
2023-12-07 21:36:44 +02:00
committed by GitHub
parent 7aa1e58301
commit 37fdbc3aaa
5 changed files with 11 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
libs/remote_storage/src/local_fs.rs
View File

@@ -260,7 +260,7 @@ impl RemoteStorage for LocalFs {
let mut buffer_to_read = data.take(from_size_bytes);
// alternatively we could just write the bytes to a file, but local_fs is a testing utility
let bytes_read = io::copy(&mut buffer_to_read, &mut destination)
let bytes_read = io::copy_buf(&mut buffer_to_read, &mut destination)
.await
.with_context(|| {
format!(

3
pageserver/src/tenant/remote_timeline_client.rs
View File

@@ -254,6 +254,9 @@ pub(crate) const FAILED_UPLOAD_WARN_THRESHOLD: u32 = 3;
pub(crate) const INITDB_PATH: &str = "initdb.tar.zst";
/// Default buffer size when interfacing with [`tokio::fs::File`].
const BUFFER_SIZE: usize = 32 * 1024;
pub enum MaybeDeletedIndexPart {
IndexPart(IndexPart),
Deleted(IndexPart),

2
pageserver/src/tenant/remote_timeline_client/download.rs
View File

@@ -90,7 +90,7 @@ pub async fn download_layer_file<'a>(
.map_err(DownloadError::Other)?;
let mut destination_file =
tokio::io::BufWriter::with_capacity(8 * 1024, destination_file);
tokio::io::BufWriter::with_capacity(super::BUFFER_SIZE, destination_file);
let mut reader = tokio_util::io::StreamReader::new(download.download_stream);

2
pageserver/src/tenant/remote_timeline_client/upload.rs
View File

@@ -105,7 +105,7 @@ pub(super) async fn upload_timeline_layer<'a>(
let fs_size = usize::try_from(fs_size)
.with_context(|| format!("convert {source_path:?} size {fs_size} usize"))?;
let reader = tokio_util::io::ReaderStream::with_capacity(source_file, 8 * 1024);
let reader = tokio_util::io::ReaderStream::with_capacity(source_file, super::BUFFER_SIZE);
storage
.upload(reader, fs_size, &storage_path, None)

7
safekeeper/src/wal_backup.rs
View File

@@ -35,6 +35,9 @@ use once_cell::sync::OnceCell;
const UPLOAD_FAILURE_RETRY_MIN_MS: u64 = 10;
const UPLOAD_FAILURE_RETRY_MAX_MS: u64 = 5000;
/// Default buffer size when interfacing with [`tokio::fs::File`].
const BUFFER_SIZE: usize = 32 * 1024;
/// Check whether wal backup is required for timeline. If yes, mark that launcher is
/// aware of current status and return the timeline.
async fn is_wal_backup_required(ttid: TenantTimelineId) -> Option<Arc<Timeline>> {
@@ -498,7 +501,7 @@ async fn backup_object(
.await
.with_context(|| format!("Failed to open file {source_file:?} for wal backup"))?;
let file = tokio_util::io::ReaderStream::with_capacity(file, 8 * 1024);
let file = tokio_util::io::ReaderStream::with_capacity(file, BUFFER_SIZE);
storage.upload_storage_object(file, size, target_file).await
}
@@ -524,7 +527,7 @@ pub async fn read_object(
let reader = tokio_util::io::StreamReader::new(download.download_stream);
let reader = tokio::io::BufReader::with_capacity(8 * 1024, reader);
let reader = tokio::io::BufReader::with_capacity(BUFFER_SIZE, reader);
Ok(Box::pin(reader))
}