rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-05-29 11:00:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: use larger buffers for remote storage (#6069)

Browse Source 
Currently using 8kB buffers, raise that to 32kB to hopefully 1/4 of
`spawn_blocking` usage. Also a drive-by fixing of last `tokio::io::copy`
to `tokio::io::copy_buf`.
This commit is contained in:
Joonas Koivunen
2023-12-07 21:36:44 +02:00
committed by GitHub
parent 7aa1e58301
commit 37fdbc3aaa
5 changed files with 11 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
pageserver/src/tenant/remote_timeline_client.rs
View File

@@ -254,6 +254,9 @@ pub(crate) const FAILED_UPLOAD_WARN_THRESHOLD: u32 = 3;
pub(crate) const INITDB_PATH: &str = "initdb.tar.zst";
/// Default buffer size when interfacing with [`tokio::fs::File`].
const BUFFER_SIZE: usize = 32 * 1024;
pub enum MaybeDeletedIndexPart {
IndexPart(IndexPart),
Deleted(IndexPart),

2
pageserver/src/tenant/remote_timeline_client/download.rs
View File

@@ -90,7 +90,7 @@ pub async fn download_layer_file<'a>(
.map_err(DownloadError::Other)?;
let mut destination_file =
tokio::io::BufWriter::with_capacity(8 * 1024, destination_file);
tokio::io::BufWriter::with_capacity(super::BUFFER_SIZE, destination_file);
let mut reader = tokio_util::io::StreamReader::new(download.download_stream);

2
pageserver/src/tenant/remote_timeline_client/upload.rs
View File

@@ -105,7 +105,7 @@ pub(super) async fn upload_timeline_layer<'a>(
let fs_size = usize::try_from(fs_size)
.with_context(|| format!("convert {source_path:?} size {fs_size} usize"))?;
let reader = tokio_util::io::ReaderStream::with_capacity(source_file, 8 * 1024);
let reader = tokio_util::io::ReaderStream::with_capacity(source_file, super::BUFFER_SIZE);
storage
.upload(reader, fs_size, &storage_path, None)