diff --git a/pageserver/benches/bench_ingest.rs b/pageserver/benches/bench_ingest.rs index 72cbb6beab..0747e426c2 100644 --- a/pageserver/benches/bench_ingest.rs +++ b/pageserver/benches/bench_ingest.rs @@ -164,12 +164,10 @@ fn criterion_benchmark(c: &mut Criterion) { let conf: &'static PageServerConf = Box::leak(Box::new( pageserver::config::PageServerConf::dummy_conf(temp_dir.path().to_path_buf()), )); - virtual_file::init( - 16384, - virtual_file::io_engine_for_bench(), - pageserver_api::config::defaults::DEFAULT_IO_BUFFER_ALIGNMENT, - ); - page_cache::init(conf.page_cache_size); + + let align = pageserver_api::config::defaults::DEFAULT_IO_BUFFER_ALIGNMENT; + virtual_file::init(16384, virtual_file::io_engine_for_bench(), align); + page_cache::init(conf.page_cache_size, align); { let mut group = c.benchmark_group("ingest-small-values"); diff --git a/pageserver/ctl/src/layer_map_analyzer.rs b/pageserver/ctl/src/layer_map_analyzer.rs index adc090823d..901e5f8a5a 100644 --- a/pageserver/ctl/src/layer_map_analyzer.rs +++ b/pageserver/ctl/src/layer_map_analyzer.rs @@ -151,13 +151,10 @@ pub(crate) async fn main(cmd: &AnalyzeLayerMapCmd) -> Result<()> { let max_holes = cmd.max_holes.unwrap_or(DEFAULT_MAX_HOLES); let ctx = RequestContext::new(TaskKind::DebugTool, DownloadBehavior::Error); + let align = pageserver_api::config::defaults::DEFAULT_IO_BUFFER_ALIGNMENT; // Initialize virtual_file (file desriptor cache) and page cache which are needed to access layer persistent B-Tree. - pageserver::virtual_file::init( - 10, - virtual_file::api::IoEngineKind::StdFs, - pageserver_api::config::defaults::DEFAULT_IO_BUFFER_ALIGNMENT, - ); - pageserver::page_cache::init(100); + pageserver::virtual_file::init(10, virtual_file::api::IoEngineKind::StdFs, align); + pageserver::page_cache::init(100, align); let mut total_delta_layers = 0usize; let mut total_image_layers = 0usize; diff --git a/pageserver/ctl/src/layers.rs b/pageserver/ctl/src/layers.rs index dd753398e2..8851e3ab10 100644 --- a/pageserver/ctl/src/layers.rs +++ b/pageserver/ctl/src/layers.rs @@ -59,8 +59,9 @@ pub(crate) enum LayerCmd { async fn read_delta_file(path: impl AsRef, ctx: &RequestContext) -> Result<()> { let path = Utf8Path::from_path(path.as_ref()).expect("non-Unicode path"); - virtual_file::init(10, virtual_file::api::IoEngineKind::StdFs, 1); - page_cache::init(100); + let align = pageserver_api::config::defaults::DEFAULT_IO_BUFFER_ALIGNMENT; + virtual_file::init(10, virtual_file::api::IoEngineKind::StdFs, align); + page_cache::init(100, align); let file = VirtualFile::open(path, ctx).await?; let file_id = page_cache::next_file_id(); let block_reader = FileBlockReader::new(&file, file_id); @@ -190,12 +191,10 @@ pub(crate) async fn main(cmd: &LayerCmd) -> Result<()> { new_tenant_id, new_timeline_id, } => { - pageserver::virtual_file::init( - 10, - virtual_file::api::IoEngineKind::StdFs, - pageserver_api::config::defaults::DEFAULT_IO_BUFFER_ALIGNMENT, - ); - pageserver::page_cache::init(100); + let align = pageserver_api::config::defaults::DEFAULT_IO_BUFFER_ALIGNMENT; + + pageserver::virtual_file::init(10, virtual_file::api::IoEngineKind::StdFs, align); + pageserver::page_cache::init(100, align); let ctx = RequestContext::new(TaskKind::DebugTool, DownloadBehavior::Error); diff --git a/pageserver/ctl/src/main.rs b/pageserver/ctl/src/main.rs index cf001ef0d5..380280358e 100644 --- a/pageserver/ctl/src/main.rs +++ b/pageserver/ctl/src/main.rs @@ -205,12 +205,9 @@ fn read_pg_control_file(control_file_path: &Utf8Path) -> anyhow::Result<()> { async fn print_layerfile(path: &Utf8Path) -> anyhow::Result<()> { // Basic initialization of things that don't change after startup - virtual_file::init( - 10, - virtual_file::api::IoEngineKind::StdFs, - DEFAULT_IO_BUFFER_ALIGNMENT, - ); - page_cache::init(100); + let align = DEFAULT_IO_BUFFER_ALIGNMENT; + virtual_file::init(10, virtual_file::api::IoEngineKind::StdFs, align); + page_cache::init(100, align); let ctx = RequestContext::new(TaskKind::DebugTool, DownloadBehavior::Error); dump_layerfile_from_path(path, true, &ctx).await } diff --git a/pageserver/src/page_cache.rs b/pageserver/src/page_cache.rs index f386c825b8..adb2d7bd4d 100644 --- a/pageserver/src/page_cache.rs +++ b/pageserver/src/page_cache.rs @@ -82,6 +82,7 @@ use once_cell::sync::OnceCell; use crate::{ context::RequestContext, metrics::{page_cache_eviction_metrics, PageCacheSizeMetrics}, + virtual_file::{self, dio::IoBufferMut}, }; static PAGE_CACHE: OnceCell = OnceCell::new(); @@ -90,8 +91,8 @@ const TEST_PAGE_CACHE_SIZE: usize = 50; /// /// Initialize the page cache. This must be called once at page server startup. /// -pub fn init(size: usize) { - if PAGE_CACHE.set(PageCache::new(size)).is_err() { +pub fn init(size: usize, align: usize) { + if PAGE_CACHE.set(PageCache::new(size, align)).is_err() { panic!("page cache already initialized"); } } @@ -106,7 +107,12 @@ pub fn get() -> &'static PageCache { // page cache is usable in unit tests. // if cfg!(test) { - PAGE_CACHE.get_or_init(|| PageCache::new(TEST_PAGE_CACHE_SIZE)) + PAGE_CACHE.get_or_init(|| { + PageCache::new( + TEST_PAGE_CACHE_SIZE, + virtual_file::get_io_buffer_alignment(), + ) + }) } else { PAGE_CACHE.get().expect("page cache not initialized") } @@ -637,13 +643,10 @@ impl PageCache { /// Initialize a new page cache /// /// This should be called only once at page server startup. - fn new(num_pages: usize) -> Self { + fn new(num_pages: usize, align: usize) -> Self { assert!(num_pages > 0, "page cache size must be > 0"); - // We could use Vec::leak here, but that potentially also leaks - // uninitialized reserved capacity. With into_boxed_slice and Box::leak - // this is avoided. - let page_buffer = Box::leak(vec![0u8; num_pages * PAGE_SZ].into_boxed_slice()); + let page_buffer = IoBufferMut::with_capacity_aligned(num_pages * PAGE_SZ, align).leak(); let size_metrics = &crate::metrics::PAGE_CACHE_SIZE; size_metrics.max_bytes.set_page_sz(num_pages); diff --git a/pageserver/src/virtual_file.rs b/pageserver/src/virtual_file.rs index 7a7491eaac..670544a555 100644 --- a/pageserver/src/virtual_file.rs +++ b/pageserver/src/virtual_file.rs @@ -415,7 +415,7 @@ impl VirtualFile { open_options.open(path_ref.as_std_path()).await? }); - // Strip all options other than read and write. + // Strip all options other than read and write (O_DIRECT). // // It would perhaps be nicer to check just for the read and write flags // explicitly, but OpenOptions doesn't contain any functions to read flags, diff --git a/pageserver/src/virtual_file/dio.rs b/pageserver/src/virtual_file/dio.rs index ab7358ef4b..11a36d5202 100644 --- a/pageserver/src/virtual_file/dio.rs +++ b/pageserver/src/virtual_file/dio.rs @@ -6,7 +6,7 @@ use std::{ cmp, mem::{ManuallyDrop, MaybeUninit}, ops::{Deref, DerefMut}, - ptr::NonNull, + ptr::{addr_of_mut, NonNull}, }; use bytes::buf::UninitSlice; @@ -175,6 +175,11 @@ impl IoBufferMut { self.ptr = ptr; self.capacity = cap; } + + pub fn leak<'a>(self) -> &'a mut [u8] { + let mut buf = ManuallyDrop::new(self); + unsafe { slice::from_raw_parts_mut(buf.as_mut_ptr(), buf.len) } + } } fn capacity_overflow() -> ! {