experiment(repeat, without preceding reverts) demonstrate that std-fs performs better because it hits the page cache

... by forcing each write system call to go to disk

test_bulk_insert[neon-release-pg14-tokio-epoll-uring].wal_written: 346 MB
test_bulk_insert[neon-release-pg14-tokio-epoll-uring].wal_recovery: 93.417 s

test_bulk_insert[neon-release-pg14-std-fs].wal_written: 346 MB
test_bulk_insert[neon-release-pg14-std-fs].wal_recovery: 86.009 s

=> ~8% instead of 2x difference

pageserver/src/virtual_file/open_options.rs

@@ -1,7 +1,12 @@
 //! Enum-dispatch to the `OpenOptions` type of the respective [`super::IoEngineKind`];
 
+use nix::libc::O_DSYNC;
+
 use super::io_engine::IoEngine;
-use std::{os::fd::OwnedFd, path::Path};
+use std::{
+    os::{fd::OwnedFd, unix::fs::OpenOptionsExt},
+    path::Path,
+};
 
 #[derive(Debug, Clone)]
 pub enum OpenOptions {
@@ -94,12 +99,14 @@ impl OpenOptions {
     }
 
     pub(in crate::virtual_file) async fn open(&self, path: &Path) -> std::io::Result<OwnedFd> {
-        match self {
+        let mut options = self.clone();
+        options.custom_flags(O_DSYNC); // disk latency
+        match options {
             OpenOptions::StdFs(x) => x.open(path).map(|file| file.into()),
             #[cfg(target_os = "linux")]
             OpenOptions::TokioEpollUring(x) => {
                 let system = tokio_epoll_uring::thread_local_system().await;
-                system.open(path, x).await.map_err(|e| match e {
+                system.open(path, &x).await.map_err(|e| match e {
                     tokio_epoll_uring::Error::Op(e) => e,
                     tokio_epoll_uring::Error::System(system) => {
                         std::io::Error::new(std::io::ErrorKind::Other, system)

test_runner/fixtures/pageserver/utils.py

@@ -200,7 +200,7 @@ def wait_for_last_record_lsn(
     lsn: Lsn,
 ) -> Lsn:
     """waits for pageserver to catch up to a certain lsn, returns the last observed lsn."""
-    for i in range(100):
+    for i in range(10000):
         current_lsn = last_record_lsn(pageserver_http, tenant, timeline)
         if current_lsn >= lsn:
             return current_lsn