From e7e1df2a797579c31f54eccc4c35150e34cb9628 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 29 Aug 2023 12:24:30 +0000 Subject: [PATCH] tokio_epoll_uring for read path --- Cargo.lock | 81 +++++++++++++++++++++++++++++----- pageserver/Cargo.toml | 1 + pageserver/src/virtual_file.rs | 46 +++++++++++++++---- 3 files changed, 108 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c78b7605b8..263664485f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1996,6 +1996,26 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "io-uring" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd1e1a01cfb924fd8c5c43b6827965db394f5a3a16c599ce03452266e1cf984c" +dependencies = [ + "bitflags", + "libc", +] + +[[package]] +name = "io-uring" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "141a0f4546a50b2ed637c7a6df0d7dff45c9f41523254996764461c8ae0d9424" +dependencies = [ + "bitflags", + "libc", +] + [[package]] name = "ipnet" version = "2.7.2" @@ -2095,9 +2115,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.144" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libloading" @@ -2663,6 +2683,7 @@ dependencies = [ "tenant_size_model", "thiserror", "tokio", + "tokio-epoll-uring", "tokio-io-timeout", "tokio-postgres", "tokio-tar", @@ -2836,9 +2857,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -3725,6 +3746,12 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.1.0" @@ -4296,18 +4323,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b" dependencies = [ "proc-macro2", "quote", @@ -4392,22 +4419,38 @@ dependencies = [ [[package]] name = "tokio" -version = "1.28.1" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ - "autocfg", + "backtrace", "bytes", "libc", "mio", "num_cpus", + "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2 0.4.9", + "socket2 0.5.3", "tokio-macros", "windows-sys 0.48.0", ] +[[package]] +name = "tokio-epoll-uring" +version = "0.1.0" +dependencies = [ + "futures", + "io-uring 0.6.1", + "once_cell", + "scopeguard", + "thiserror", + "tokio", + "tokio-uring", + "tokio-util", + "tracing", +] + [[package]] name = "tokio-io-timeout" version = "1.2.0" @@ -4547,6 +4590,20 @@ dependencies = [ "tungstenite 0.20.0", ] +[[package]] +name = "tokio-uring" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d5e02bb137e030b3a547c65a3bd2f1836d66a97369fdcc69034002b10e155ef" +dependencies = [ + "io-uring 0.5.13", + "libc", + "scoped-tls", + "slab", + "socket2 0.4.9", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.8" diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index bbdd8b1e99..5cfc0f9bba 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -80,6 +80,7 @@ enum-map.workspace = true enumset.workspace = true strum.workspace = true strum_macros.workspace = true +tokio-epoll-uring = { path = "/home/admin/tokio-epoll-uring/tokio-epoll-uring" } [dev-dependencies] criterion.workspace = true diff --git a/pageserver/src/virtual_file.rs b/pageserver/src/virtual_file.rs index 79de388cc2..922b7a52c9 100644 --- a/pageserver/src/virtual_file.rs +++ b/pageserver/src/virtual_file.rs @@ -227,16 +227,46 @@ impl VirtualFile { panic!("mut put self.handle back") } }; - let ((file, write_guard), res) = tokio::task::spawn_blocking(move || { - let res = file.read_exact_at(write_guard.as_mut(), offset); - ((file, write_guard), res) - }) - .await - .expect("spawn_blocking"); - let replaced = self.handle.lock().unwrap().replace(file); + let system = tokio_epoll_uring::thread_local_system().await; + struct PageWriteGuardBuf { + buf: PageWriteGuard<'static>, + init_up_to: usize, + } + unsafe impl tokio_epoll_uring::IoBuf for PageWriteGuardBuf { + fn stable_ptr(&self) -> *const u8 { + self.buf.as_ptr() + } + fn bytes_init(&self) -> usize { + self.init_up_to + } + fn bytes_total(&self) -> usize { + self.buf.len() + } + } + unsafe impl tokio_epoll_uring::IoBufMut for PageWriteGuardBuf { + fn stable_mut_ptr(&mut self) -> *mut u8 { + self.buf.as_mut_ptr() + } + + unsafe fn set_init(&mut self, pos: usize) { + assert!(pos <= self.buf.len()); + self.init_up_to = pos; + } + } + let buf = PageWriteGuardBuf { + buf: write_guard, + init_up_to: 0, + }; + let ((file, buf), res) = system.read(file.into(), offset, buf).await; + let PageWriteGuardBuf { buf: write_guard, init_up_to } = buf; + if let Ok(num_read) = res { + assert!(init_up_to <= num_read); + } + let replaced = self.handle.lock().unwrap().replace(File::from(file)); assert!(replaced.is_none()); put_back.store(true, std::sync::atomic::Ordering::Relaxed); - res.map(|()| write_guard) + res.map(|_| write_guard) + .map_err(|e| Error::new(ErrorKind::Other, e)) } // Copied from https://doc.rust-lang.org/1.72.0/src/std/os/unix/fs.rs.html#219-235