From d66ccbae5efd7e70342337d94eeca4c1e0d6fb5b Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 13 Mar 2024 15:03:24 +0000 Subject: [PATCH] experiment(repeat, without preceding reverts) demonstrate that std-fs performs better because it hits the page cache ... by forcing each write system call to go to disk test_bulk_insert[neon-release-pg14-tokio-epoll-uring].wal_written: 346 MB test_bulk_insert[neon-release-pg14-tokio-epoll-uring].wal_recovery: 93.417 s test_bulk_insert[neon-release-pg14-std-fs].wal_written: 346 MB test_bulk_insert[neon-release-pg14-std-fs].wal_recovery: 86.009 s => ~8% instead of 2x difference --- pageserver/src/virtual_file/open_options.rs | 13 ++++++++++--- test_runner/fixtures/pageserver/utils.py | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pageserver/src/virtual_file/open_options.rs b/pageserver/src/virtual_file/open_options.rs index f75edb0bac..6952cf3194 100644 --- a/pageserver/src/virtual_file/open_options.rs +++ b/pageserver/src/virtual_file/open_options.rs @@ -1,7 +1,12 @@ //! Enum-dispatch to the `OpenOptions` type of the respective [`super::IoEngineKind`]; +use nix::libc::O_DSYNC; + use super::io_engine::IoEngine; -use std::{os::fd::OwnedFd, path::Path}; +use std::{ + os::{fd::OwnedFd, unix::fs::OpenOptionsExt}, + path::Path, +}; #[derive(Debug, Clone)] pub enum OpenOptions { @@ -94,12 +99,14 @@ impl OpenOptions { } pub(in crate::virtual_file) async fn open(&self, path: &Path) -> std::io::Result { - match self { + let mut options = self.clone(); + options.custom_flags(O_DSYNC); // disk latency + match options { OpenOptions::StdFs(x) => x.open(path).map(|file| file.into()), #[cfg(target_os = "linux")] OpenOptions::TokioEpollUring(x) => { let system = tokio_epoll_uring::thread_local_system().await; - system.open(path, x).await.map_err(|e| match e { + system.open(path, &x).await.map_err(|e| match e { tokio_epoll_uring::Error::Op(e) => e, tokio_epoll_uring::Error::System(system) => { std::io::Error::new(std::io::ErrorKind::Other, system) diff --git a/test_runner/fixtures/pageserver/utils.py b/test_runner/fixtures/pageserver/utils.py index cf64c86821..9452ee611c 100644 --- a/test_runner/fixtures/pageserver/utils.py +++ b/test_runner/fixtures/pageserver/utils.py @@ -200,7 +200,7 @@ def wait_for_last_record_lsn( lsn: Lsn, ) -> Lsn: """waits for pageserver to catch up to a certain lsn, returns the last observed lsn.""" - for i in range(100): + for i in range(10000): current_lsn = last_record_lsn(pageserver_http, tenant, timeline) if current_lsn >= lsn: return current_lsn