diff --git a/Cargo.lock b/Cargo.lock index fe8732628f..e26ba2a8c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -253,6 +253,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8ab6b55fe97976e46f91ddbed8d147d966475dc29b2032757ba47e02376fbc3" +[[package]] +name = "atomic_enum" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99e1aca718ea7b89985790c94aad72d77533063fe00bc497bb79a7c2dae6a661" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -687,13 +698,40 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core 0.4.5", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "itoa", + "matchit 0.7.3", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper 1.0.1", + "tower 0.5.2", + "tower-layer", + "tower-service", +] + [[package]] name = "axum" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d6fd624c75e18b3b4c6b9caf42b1afe24437daaee904069137d8bab077be8b8" dependencies = [ - "axum-core", + "axum-core 0.5.0", "base64 0.22.1", "bytes", "form_urlencoded", @@ -704,7 +742,7 @@ dependencies = [ "hyper 1.6.0", "hyper-util", "itoa", - "matchit", + "matchit 0.8.4", "memchr", "mime", "percent-encoding", @@ -724,6 +762,26 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 1.0.1", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-core" version = "0.5.0" @@ -750,8 +808,8 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fc6f625a1f7705c6cf62d0d070794e94668988b1c38111baeec177c715f7b" dependencies = [ - "axum", - "axum-core", + "axum 0.8.1", + "axum-core 0.5.0", "bytes", "futures-util", "headers", @@ -1086,6 +1144,25 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cbindgen" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadd868a2ce9ca38de7eeafdcec9c7065ef89b42b32f0839278d55f35c54d1ff" +dependencies = [ + "clap", + "heck 0.4.1", + "indexmap 2.9.0", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 2.0.100", + "tempfile", + "toml", +] + [[package]] name = "cc" version = "1.2.16" @@ -1212,7 +1289,7 @@ version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.100", @@ -1270,6 +1347,34 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "communicator" +version = "0.1.0" +dependencies = [ + "atomic_enum", + "axum 0.8.1", + "bytes", + "cbindgen", + "clashmap", + "http 1.1.0", + "libc", + "metrics", + "neon-shmem", + "nix 0.30.1", + "pageserver_client_grpc", + "pageserver_page_api", + "prometheus", + "prost 0.13.5", + "thiserror 1.0.69", + "tokio", + "tokio-pipe", + "tonic 0.12.3", + "tracing", + "tracing-subscriber", + "uring-common", + "utils", +] + [[package]] name = "compute_api" version = "0.1.0" @@ -1295,7 +1400,7 @@ dependencies = [ "aws-sdk-kms", "aws-sdk-s3", "aws-smithy-types", - "axum", + "axum 0.8.1", "axum-extra", "base64 0.13.1", "bytes", @@ -1936,7 +2041,7 @@ checksum = "0892a17df262a24294c382f0d5997571006e7a4348b4327557c4ff1cd4a8bccc" dependencies = [ "darling", "either", - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.100", @@ -2050,7 +2155,7 @@ name = "endpoint_storage" version = "0.0.1" dependencies = [ "anyhow", - "axum", + "axum 0.8.1", "axum-extra", "camino", "camino-tempfile", @@ -2724,6 +2829,12 @@ dependencies = [ "http 1.1.0", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "heck" version = "0.5.0" @@ -3615,6 +3726,12 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "matchit" version = "0.8.4" @@ -3660,7 +3777,7 @@ version = "0.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e6777fc80a575f9503d908c8b498782a6c3ee88a06cb416dc3941401e43b94" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.100", @@ -3819,6 +3936,17 @@ dependencies = [ "workspace_hack", ] +[[package]] +name = "neonart" +version = "0.1.0" +dependencies = [ + "crossbeam-utils", + "rand 0.9.1", + "rand_distr 0.5.1", + "spin", + "tracing", +] + [[package]] name = "never-say-never" version = "6.6.666" @@ -4252,7 +4380,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", - "axum", + "axum 0.8.1", "camino", "clap", "futures", @@ -5156,7 +5284,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ "bytes", - "heck", + "heck 0.5.0", "itertools 0.12.1", "log", "multimap", @@ -5177,7 +5305,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c1318b19085f08681016926435853bbf7858f9c082d0999b80550ff5d9abe15" dependencies = [ "bytes", - "heck", + "heck 0.5.0", "itertools 0.12.1", "log", "multimap", @@ -5837,7 +5965,7 @@ dependencies = [ "async-trait", "getrandom 0.2.11", "http 1.1.0", - "matchit", + "matchit 0.8.4", "opentelemetry", "reqwest", "reqwest-middleware", @@ -7012,7 +7140,7 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "rustversion", @@ -7437,6 +7565,16 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "tokio-pipe" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f213a84bffbd61b8fa0ba8a044b4bbe35d471d0b518867181e82bd5c15542784" +dependencies = [ + "libc", + "tokio", +] + [[package]] name = "tokio-postgres" version = "0.7.10" @@ -7631,16 +7769,25 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ + "async-stream", "async-trait", + "axum 0.7.9", "base64 0.22.1", "bytes", + "h2 0.4.4", "http 1.1.0", "http-body 1.0.0", "http-body-util", + "hyper 1.6.0", + "hyper-timeout", + "hyper-util", "percent-encoding", "pin-project", "prost 0.13.5", + "socket2", + "tokio", "tokio-stream", + "tower 0.4.13", "tower-layer", "tower-service", "tracing", @@ -7653,7 +7800,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" dependencies = [ "async-trait", - "axum", + "axum 0.8.1", "base64 0.22.1", "bytes", "flate2", @@ -7713,9 +7860,13 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", + "indexmap 1.9.3", "pin-project", "pin-project-lite", + "rand 0.8.5", + "slab", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -8202,7 +8353,7 @@ name = "vm_monitor" version = "0.1.0" dependencies = [ "anyhow", - "axum", + "axum 0.8.1", "cgroups-rs", "clap", "futures", @@ -8695,8 +8846,8 @@ dependencies = [ "ahash", "anstream", "anyhow", - "axum", - "axum-core", + "axum 0.8.1", + "axum-core 0.5.0", "base64 0.13.1", "base64 0.21.7", "base64ct", diff --git a/pgxn/neon/communicator/src/integrated_cache.rs b/pgxn/neon/communicator/src/integrated_cache.rs index 78a99390e2..4c65af6bdb 100644 --- a/pgxn/neon/communicator/src/integrated_cache.rs +++ b/pgxn/neon/communicator/src/integrated_cache.rs @@ -29,7 +29,7 @@ use utils::lsn::{AtomicLsn, Lsn}; use crate::file_cache::INVALID_CACHE_BLOCK; use crate::file_cache::{CacheBlock, FileCache}; -use pageserver_page_api::model::RelTag; +use pageserver_page_api::RelTag; use metrics::{IntCounter, IntGauge}; diff --git a/pgxn/neon/communicator/src/neon_request.rs b/pgxn/neon/communicator/src/neon_request.rs index c90876347f..666f628112 100644 --- a/pgxn/neon/communicator/src/neon_request.rs +++ b/pgxn/neon/communicator/src/neon_request.rs @@ -4,7 +4,7 @@ type COid = u32; // This conveniently matches PG_IOV_MAX pub const MAX_GETPAGEV_PAGES: usize = 32; -use pageserver_page_api::model; +use pageserver_page_api as page_api; #[repr(C)] #[derive(Copy, Clone, Debug)] @@ -12,7 +12,7 @@ pub enum NeonIORequest { Empty, // Read requests. These are C-friendly variants of the corresponding structs in - // pageserver_page_api::model. + // pageserver_page_api. RelExists(CRelExistsRequest), RelSize(CRelSizeRequest), GetPageV(CGetPageVRequest), @@ -236,8 +236,8 @@ pub struct CRelUnlinkRequest { } impl CRelExistsRequest { - pub fn reltag(&self) -> model::RelTag { - model::RelTag { + pub fn reltag(&self) -> page_api::RelTag { + page_api::RelTag { spc_oid: self.spc_oid, db_oid: self.db_oid, rel_number: self.rel_number, @@ -247,8 +247,8 @@ impl CRelExistsRequest { } impl CRelSizeRequest { - pub fn reltag(&self) -> model::RelTag { - model::RelTag { + pub fn reltag(&self) -> page_api::RelTag { + page_api::RelTag { spc_oid: self.spc_oid, db_oid: self.db_oid, rel_number: self.rel_number, @@ -258,8 +258,8 @@ impl CRelSizeRequest { } impl CGetPageVRequest { - pub fn reltag(&self) -> model::RelTag { - model::RelTag { + pub fn reltag(&self) -> page_api::RelTag { + page_api::RelTag { spc_oid: self.spc_oid, db_oid: self.db_oid, rel_number: self.rel_number, @@ -269,8 +269,8 @@ impl CGetPageVRequest { } impl CPrefetchVRequest { - pub fn reltag(&self) -> model::RelTag { - model::RelTag { + pub fn reltag(&self) -> page_api::RelTag { + page_api::RelTag { spc_oid: self.spc_oid, db_oid: self.db_oid, rel_number: self.rel_number, @@ -280,8 +280,8 @@ impl CPrefetchVRequest { } impl CWritePageRequest { - pub fn reltag(&self) -> model::RelTag { - model::RelTag { + pub fn reltag(&self) -> page_api::RelTag { + page_api::RelTag { spc_oid: self.spc_oid, db_oid: self.db_oid, rel_number: self.rel_number, @@ -291,8 +291,8 @@ impl CWritePageRequest { } impl CRelExtendRequest { - pub fn reltag(&self) -> model::RelTag { - model::RelTag { + pub fn reltag(&self) -> page_api::RelTag { + page_api::RelTag { spc_oid: self.spc_oid, db_oid: self.db_oid, rel_number: self.rel_number, @@ -302,8 +302,8 @@ impl CRelExtendRequest { } impl CRelZeroExtendRequest { - pub fn reltag(&self) -> model::RelTag { - model::RelTag { + pub fn reltag(&self) -> page_api::RelTag { + page_api::RelTag { spc_oid: self.spc_oid, db_oid: self.db_oid, rel_number: self.rel_number, @@ -313,8 +313,8 @@ impl CRelZeroExtendRequest { } impl CRelCreateRequest { - pub fn reltag(&self) -> model::RelTag { - model::RelTag { + pub fn reltag(&self) -> page_api::RelTag { + page_api::RelTag { spc_oid: self.spc_oid, db_oid: self.db_oid, rel_number: self.rel_number, @@ -324,8 +324,8 @@ impl CRelCreateRequest { } impl CRelTruncateRequest { - pub fn reltag(&self) -> model::RelTag { - model::RelTag { + pub fn reltag(&self) -> page_api::RelTag { + page_api::RelTag { spc_oid: self.spc_oid, db_oid: self.db_oid, rel_number: self.rel_number, @@ -335,8 +335,8 @@ impl CRelTruncateRequest { } impl CRelUnlinkRequest { - pub fn reltag(&self) -> model::RelTag { - model::RelTag { + pub fn reltag(&self) -> page_api::RelTag { + page_api::RelTag { spc_oid: self.spc_oid, db_oid: self.db_oid, rel_number: self.rel_number, diff --git a/pgxn/neon/communicator/src/worker_process/in_progress_ios.rs b/pgxn/neon/communicator/src/worker_process/in_progress_ios.rs index 71560b8b46..845479fcda 100644 --- a/pgxn/neon/communicator/src/worker_process/in_progress_ios.rs +++ b/pgxn/neon/communicator/src/worker_process/in_progress_ios.rs @@ -7,13 +7,13 @@ use tokio::sync::{Mutex, OwnedMutexGuard}; use clashmap::ClashMap; use clashmap::Entry; -use pageserver_page_api::model; +use pageserver_page_api::RelTag; #[derive(Clone, Eq, Hash, PartialEq)] pub enum RequestInProgressKey { Db(u32), - Rel(model::RelTag), - Block(model::RelTag, u32), + Rel(RelTag), + Block(RelTag, u32), } pub type RequestInProgressTable = MutexHashSet; diff --git a/pgxn/neon/communicator/src/worker_process/main_loop.rs b/pgxn/neon/communicator/src/worker_process/main_loop.rs index 5ab18c8eb8..9f36bf9fd5 100644 --- a/pgxn/neon/communicator/src/worker_process/main_loop.rs +++ b/pgxn/neon/communicator/src/worker_process/main_loop.rs @@ -13,7 +13,7 @@ use crate::neon_request::{CGetPageVRequest, CPrefetchVRequest}; use crate::neon_request::{NeonIORequest, NeonIOResult}; use crate::worker_process::in_progress_ios::{RequestInProgressKey, RequestInProgressTable}; use pageserver_client_grpc::PageserverClient; -use pageserver_page_api::model; +use pageserver_page_api as page_api; use metrics::{IntCounter, IntCounterVec}; @@ -224,10 +224,10 @@ impl<'t> CommunicatorWorkerProcessStruct<'t> { } } - fn request_lsns(&self, not_modified_since_lsn: Lsn) -> model::ReadLsn { - model::ReadLsn { + fn request_lsns(&self, not_modified_since_lsn: Lsn) -> page_api::ReadLsn { + page_api::ReadLsn { request_lsn: get_request_lsn(), - not_modified_since_lsn, + not_modified_since_lsn: Some(not_modified_since_lsn), } } @@ -252,7 +252,7 @@ impl<'t> CommunicatorWorkerProcessStruct<'t> { match self .pageserver_client - .process_check_rel_exists_request(&model::CheckRelExistsRequest { + .process_check_rel_exists_request(&page_api::CheckRelExistsRequest { read_lsn: self.request_lsns(not_modified_since), rel, }) @@ -286,7 +286,7 @@ impl<'t> CommunicatorWorkerProcessStruct<'t> { let read_lsn = self.request_lsns(not_modified_since); match self .pageserver_client - .process_get_rel_size_request(&model::GetRelSizeRequest { + .process_get_rel_size_request(&page_api::GetRelSizeRequest { read_lsn, rel: rel.clone(), }) @@ -339,7 +339,7 @@ impl<'t> CommunicatorWorkerProcessStruct<'t> { match self .pageserver_client - .process_get_dbsize_request(&model::GetDbSizeRequest { + .process_get_dbsize_request(&page_api::GetDbSizeRequest { read_lsn: self.request_lsns(not_modified_since), db_oid: req.db_oid, }) @@ -462,12 +462,12 @@ impl<'t> CommunicatorWorkerProcessStruct<'t> { for (blkno, _lsn, dest, _guard) in cache_misses.iter() { match self .pageserver_client - .get_page(&model::GetPageRequest { + .get_page(page_api::GetPageRequest { request_id: self.next_request_id.fetch_add(1, Ordering::Relaxed), - request_class: model::GetPageClass::Normal, + request_class: page_api::GetPageClass::Normal, read_lsn: self.request_lsns(not_modified_since), rel: rel.clone(), - block_number: vec![*blkno], + block_numbers: vec![*blkno], }) .await { @@ -540,12 +540,12 @@ impl<'t> CommunicatorWorkerProcessStruct<'t> { for (blkno, _lsn, _guard) in cache_misses.iter() { match self .pageserver_client - .get_page(&model::GetPageRequest { + .get_page(page_api::GetPageRequest { request_id: self.next_request_id.fetch_add(1, Ordering::Relaxed), - request_class: model::GetPageClass::Prefetch, + request_class: page_api::GetPageClass::Prefetch, read_lsn: self.request_lsns(not_modified_since), rel: rel.clone(), - block_number: vec![*blkno], + block_numbers: vec![*blkno], }) .await {