From 2f9cc9a11ee03efef9854336fa2dc2736c1de811 Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Tue, 29 Jul 2025 17:35:47 +0100 Subject: [PATCH] tests: remove tests that depend on libpq --- Cargo.lock | 420 +---------------------- storage_controller/Cargo.toml | 10 - storage_controller/src/hadron_queries.rs | 416 ---------------------- workspace_hack/Cargo.toml | 68 +--- 4 files changed, 10 insertions(+), 904 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f40384e3e1..d465fe6f21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -247,15 +247,6 @@ dependencies = [ "syn 2.0.100", ] -[[package]] -name = "atoi" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" -dependencies = [ - "num-traits", -] - [[package]] name = "atomic-take" version = "1.1.0" @@ -1550,21 +1541,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - [[package]] name = "crc32c" version = "0.6.8" @@ -1659,15 +1635,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "crossbeam-queue" -version = "0.3.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-utils" version = "0.8.19" @@ -1915,7 +1882,6 @@ dependencies = [ "chrono", "diesel_derives", "itoa", - "pq-sys", "serde_json", "uuid", ] @@ -2006,12 +1972,6 @@ dependencies = [ "const-random", ] -[[package]] -name = "dotenvy" -version = "0.15.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" - [[package]] name = "dsl_auto_type" version = "0.1.1" @@ -2085,9 +2045,6 @@ name = "either" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" -dependencies = [ - "serde", -] [[package]] name = "elliptic-curve" @@ -2261,17 +2218,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "etcetera" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" -dependencies = [ - "cfg-if", - "home", - "windows-sys 0.48.0", -] - [[package]] name = "event-listener" version = "2.5.3" @@ -2409,27 +2355,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.1" @@ -2511,17 +2436,6 @@ dependencies = [ "futures-util", ] -[[package]] -name = "futures-intrusive" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" -dependencies = [ - "futures-core", - "lock_api", - "parking_lot 0.12.1", -] - [[package]] name = "futures-io" version = "0.3.31" @@ -2823,11 +2737,6 @@ name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] [[package]] name = "hashlink" @@ -2838,15 +2747,6 @@ dependencies = [ "hashbrown 0.14.5", ] -[[package]] -name = "hashlink" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" -dependencies = [ - "hashbrown 0.15.2", -] - [[package]] name = "hdrhistogram" version = "7.5.4" @@ -2927,15 +2827,6 @@ dependencies = [ "never-say-never", ] -[[package]] -name = "hkdf" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" -dependencies = [ - "hmac", -] - [[package]] name = "hmac" version = "0.12.1" @@ -2945,15 +2836,6 @@ dependencies = [ "digest", ] -[[package]] -name = "home" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" -dependencies = [ - "windows-sys 0.59.0", -] - [[package]] name = "hostname" version = "0.4.0" @@ -3203,22 +3085,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper 1.6.0", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.16" @@ -4084,23 +3950,6 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" -[[package]] -name = "native-tls" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "neon-shmem" version = "0.1.0" @@ -4378,50 +4227,12 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" -[[package]] -name = "openssl" -version = "0.10.73" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" -dependencies = [ - "bitflags 2.8.0", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-sys" -version = "0.9.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "opentelemetry" version = "0.30.0" @@ -4681,7 +4492,7 @@ dependencies = [ "enumset", "fail", "futures", - "hashlink 0.9.1", + "hashlink", "hex", "hex-literal", "http 1.3.1", @@ -5336,65 +5147,6 @@ dependencies = [ "workspace_hack", ] -[[package]] -name = "postgresql_archive" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b10a6a63e5f4ccf5d99d7d5db0829aec30a5e84b5c80883f6561c77a7521a6" -dependencies = [ - "async-trait", - "flate2", - "futures-util", - "hex", - "num-format", - "regex-lite", - "reqwest", - "reqwest-middleware", - "reqwest-retry", - "reqwest-tracing", - "semver", - "serde", - "serde_json", - "sha2", - "tar", - "target-triple", - "tempfile", - "thiserror 2.0.12", - "tracing", - "url", -] - -[[package]] -name = "postgresql_commands" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cc8653c31df5ffcd3bc114641dfae5390404d6e22bc21fd7473760f22cdc20b" -dependencies = [ - "thiserror 2.0.12", - "tokio", - "tracing", -] - -[[package]] -name = "postgresql_embedded" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4386829807c8dfbc190f8e53b91d2080f0f604bd4b9f0831281ccfbc06998123" -dependencies = [ - "anyhow", - "postgresql_archive", - "postgresql_commands", - "rand 0.9.1", - "semver", - "sqlx", - "target-triple", - "tempfile", - "thiserror 2.0.12", - "tokio", - "tracing", - "url", -] - [[package]] name = "posthog_client_lite" version = "0.1.0" @@ -5467,15 +5219,6 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" -[[package]] -name = "pq-sys" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6cc05d7ea95200187117196eee9edd0644424911821aeb28a18ce60ea0b8793" -dependencies = [ - "vcpkg", -] - [[package]] name = "pq_proto" version = "0.1.0" @@ -6258,11 +6001,9 @@ dependencies = [ "http-body-util", "hyper 1.6.0", "hyper-rustls 0.27.7", - "hyper-tls", "hyper-util", "js-sys", "log", - "native-tls", "percent-encoding", "pin-project-lite", "quinn", @@ -6274,7 +6015,6 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.1", "tokio", - "tokio-native-tls", "tokio-rustls 0.26.2", "tokio-util", "tower", @@ -7255,9 +6995,6 @@ name = "smallvec" version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" -dependencies = [ - "serde", -] [[package]] name = "smol_str" @@ -7323,125 +7060,6 @@ dependencies = [ "der 0.7.8", ] -[[package]] -name = "sqlx" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" -dependencies = [ - "sqlx-core", - "sqlx-macros", - "sqlx-postgres", -] - -[[package]] -name = "sqlx-core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" -dependencies = [ - "base64 0.22.1", - "bytes", - "crc", - "crossbeam-queue", - "either", - "event-listener 5.4.0", - "futures-core", - "futures-intrusive", - "futures-io", - "futures-util", - "hashbrown 0.15.2", - "hashlink 0.10.0", - "indexmap 2.10.0", - "log", - "memchr", - "native-tls", - "once_cell", - "percent-encoding", - "serde", - "serde_json", - "sha2", - "smallvec", - "thiserror 2.0.12", - "tokio", - "tokio-stream", - "tracing", - "url", -] - -[[package]] -name = "sqlx-macros" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" -dependencies = [ - "proc-macro2", - "quote", - "sqlx-core", - "sqlx-macros-core", - "syn 2.0.100", -] - -[[package]] -name = "sqlx-macros-core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" -dependencies = [ - "dotenvy", - "either", - "heck 0.5.0", - "hex", - "once_cell", - "proc-macro2", - "quote", - "serde", - "serde_json", - "sha2", - "sqlx-core", - "sqlx-postgres", - "syn 2.0.100", - "tokio", - "url", -] - -[[package]] -name = "sqlx-postgres" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" -dependencies = [ - "atoi", - "base64 0.22.1", - "bitflags 2.8.0", - "byteorder", - "crc", - "dotenvy", - "etcetera", - "futures-channel", - "futures-core", - "futures-util", - "hex", - "hkdf", - "hmac", - "home", - "itoa", - "log", - "md-5", - "memchr", - "once_cell", - "rand 0.8.5", - "serde", - "serde_json", - "sha2", - "smallvec", - "sqlx-core", - "stringprep", - "thiserror 2.0.12", - "tracing", - "whoami", -] - [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -7519,8 +7137,6 @@ dependencies = [ "pageserver_api", "pageserver_client", "postgres_connection", - "postgresql_archive", - "postgresql_embedded", "posthog_client_lite", "rand 0.9.1", "regex", @@ -7787,12 +7403,6 @@ dependencies = [ "xattr", ] -[[package]] -name = "target-triple" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac9aa371f599d22256307c24a9d748c041e548cbf599f35d890f9d365361790" - [[package]] name = "tempfile" version = "3.20.0" @@ -8104,16 +7714,6 @@ dependencies = [ "syn 2.0.100", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-postgres" version = "0.7.10" @@ -8831,12 +8431,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version_check" version = "0.9.4" @@ -9456,7 +9050,6 @@ dependencies = [ "ahash", "anstream", "anyhow", - "arrayvec", "axum", "axum-core", "base64 0.21.7", @@ -9467,21 +9060,18 @@ dependencies = [ "chrono", "clap", "clap_builder", - "concurrent-queue", "const-oid", "crossbeam-epoch", "crossbeam-utils", "crypto-bigint 0.5.5", "der 0.7.8", "deranged", - "diesel", "digest", "ecdsa 0.16.9", "either", "elliptic-curve 0.13.8", "env_filter", "env_logger", - "event-listener 5.4.0", "fail", "form_urlencoded", "futures-channel", @@ -9494,7 +9084,6 @@ dependencies = [ "getrandom 0.2.11", "half", "hashbrown 0.14.5", - "hashbrown 0.15.2", "hex", "hmac", "hyper 0.14.30", @@ -9512,18 +9101,14 @@ dependencies = [ "num", "num-bigint", "num-complex", - "num-format", "num-integer", "num-iter", "num-rational", "num-traits", "once_cell", - "opentelemetry_sdk", "p256 0.13.2", "parquet", - "percent-encoding", "portable-atomic", - "postgresql_archive", "prettyplease", "proc-macro2", "prost 0.13.5", @@ -9533,8 +9118,6 @@ dependencies = [ "regex-automata 0.4.9", "regex-syntax 0.8.5", "reqwest", - "reqwest-middleware", - "reqwest-tracing", "rustls 0.23.29", "rustls-pki-types", "rustls-webpki 0.103.4", @@ -9567,7 +9150,6 @@ dependencies = [ "tracing-log", "tracing-subscriber", "url", - "uuid", "zeroize", "zstd", "zstd-safe", diff --git a/storage_controller/Cargo.toml b/storage_controller/Cargo.toml index 9f9e66eecd..94ea5f14ff 100644 --- a/storage_controller/Cargo.toml +++ b/storage_controller/Cargo.toml @@ -75,13 +75,3 @@ utils = { path = "../libs/utils/" } metrics = { path = "../libs/metrics/" } control_plane = { path = "../control_plane" } workspace_hack = { version = "0.1", path = "../workspace_hack" } - -[dev-dependencies] -postgresql_archive = "0.19.0" -postgresql_embedded = { version = "0.19.0", features = ["blocking"] } -diesel = { version = "2.2.6", features = [ - "postgres", - "serde_json", - "chrono", - "uuid", -] } diff --git a/storage_controller/src/hadron_queries.rs b/storage_controller/src/hadron_queries.rs index 16d16f6483..e786c2238a 100644 --- a/storage_controller/src/hadron_queries.rs +++ b/storage_controller/src/hadron_queries.rs @@ -431,419 +431,3 @@ async fn get_pageserver_connection_info( } Ok(shards) } - -#[cfg(test)] -mod test { - - use std::collections::BTreeMap; - - use super::*; - use crate::schema::hadron_safekeepers; - use diesel::PgConnection; - use diesel_migrations::{EmbeddedMigrations, MigrationHarness, embed_migrations}; - use pageserver_api::controller_api::{SCSafekeeperTimeline, TimelineSafekeeperPeer}; - use postgresql_archive::VersionReq; - use postgresql_embedded::Settings; - use postgresql_embedded::blocking::PostgreSQL; - - async fn get_embedded_pg() -> postgresql_embedded::Result { - tokio::task::spawn_blocking(|| { - let pg_install_dir = "../pg_install/16.0.0"; - // Link "pg_install/v16" -> "pg_install/16.0.0" so that it can be picked up by the postgres_embedded - // crate without needing to download anything. The postgres_embedded crate expects a specific format - // for the directory name. - let _ = std::os::unix::fs::symlink("./v16", pg_install_dir); - - let settings = Settings { - installation_dir: std::path::PathBuf::from(pg_install_dir), - username: "postgres".to_string(), - password: "password".to_string(), - // Use a 30-second timeout for database initialization to avoid flakiness in the CI environment. - timeout: Some(std::time::Duration::from_secs(30)), - version: VersionReq::parse("=16.0.0").unwrap(), - ..Default::default() - }; - let mut pg = PostgreSQL::new(settings); - - pg.setup()?; - pg.start()?; - - pg.create_database("test")?; - - Ok(pg) - }) - .await - .unwrap() - } - - pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations"); - - fn run_migrations(connection: &mut PgConnection) -> Result<(), String> { - connection.run_pending_migrations(MIGRATIONS).unwrap(); - Ok(()) - } - - fn get_test_sk_node(id: u64) -> SafeKeeperNode { - SafeKeeperNode::new( - NodeId(id), - format!("safekeeper-{id}"), - 123, - format!("safekeeper-{id}"), - 456, - ) - } - - #[tokio::test] - async fn test_safekeeper_upserts_and_list() { - let pg = get_embedded_pg().await.unwrap(); - - let connection_string = pg.settings().url("test"); - { - let mut conn = PgConnection::establish(&connection_string) - .unwrap_or_else(|_| panic!("Error connecting to {connection_string}")); - run_migrations(&mut conn).unwrap(); - } - - let mut connection = AsyncPgConnection::establish(&connection_string) - .await - .unwrap_or_else(|_| panic!("Error connecting to {connection_string}")); - - execute_sk_upsert(&mut connection, get_test_sk_node(0).to_database_row()) - .await - .unwrap(); - execute_sk_upsert(&mut connection, get_test_sk_node(1).to_database_row()) - .await - .unwrap(); - execute_sk_upsert(&mut connection, get_test_sk_node(2).to_database_row()) - .await - .unwrap(); - - // Insert an entry into the hadron_timeline_safekeepers table. - use crate::schema::hadron_timeline_safekeepers; - let timeline1_id = TimelineId::generate(); - diesel::insert_into(hadron_timeline_safekeepers::table) - .values(&HadronTimelineSafekeeper { - timeline_id: timeline1_id.to_string(), - sk_node_id: 0, - legacy_endpoint_id: None, - }) - .execute(&mut connection) - .await - .expect("Failed to insert timeline1"); - - // Test that the nodes have indeed been inserted - let sk_nodes = hadron_safekeepers::table - .load::(&mut connection) - .await - .unwrap(); - assert_eq!(sk_nodes.len(), 3); - assert_eq!(sk_nodes[0].sk_node_id, 0); - assert_eq!(sk_nodes[1].sk_node_id, 1); - assert_eq!(sk_nodes[2].sk_node_id, 2); - - // Test that we can read the nodes back out in the join query, where we pull all the Safekeepers along with their endpoints scheduled. - // There should be no endpoints in this test, verify that nothing breaks. - let sk_nodes = scan_safekeepers_and_scheduled_timelines(&mut connection) - .await - .unwrap(); - assert_eq!(sk_nodes.len(), 3); - assert_eq!(sk_nodes[&NodeId(0)].legacy_endpoints.len(), 0); - assert_eq!(sk_nodes[&NodeId(1)].legacy_endpoints.len(), 0); - assert_eq!(sk_nodes[&NodeId(2)].legacy_endpoints.len(), 0); - - // Test that only the 0th safekeeper is assigned to the timeline. - assert_eq!(sk_nodes[&NodeId(0)].timelines.len(), 1); - assert_eq!(sk_nodes[&NodeId(1)].timelines.len(), 0); - assert_eq!(sk_nodes[&NodeId(2)].timelines.len(), 0); - } - - #[tokio::test] - async fn test_idempotently_persist_or_get_existing_timeline_safekeepers() { - let pg = get_embedded_pg().await.unwrap(); - - let connection_string = pg.settings().url("test"); - { - let mut conn = PgConnection::establish(&connection_string) - .unwrap_or_else(|_| panic!("Error connecting to {connection_string}")); - run_migrations(&mut conn).unwrap(); - } - - let mut connection = AsyncPgConnection::establish(&connection_string) - .await - .unwrap_or_else(|_| panic!("Error connecting to {connection_string}")); - - // An initial call should insert the timeline safekeepers and return the inserted values. - let timeline1_id = TimelineId::generate(); - let safekeeper_ids = vec![NodeId(1), NodeId(2), NodeId(3)]; - let inserted = idempotently_persist_or_get_existing_timeline_safekeepers( - &mut connection, - timeline1_id, - &safekeeper_ids, - ) - .await - .expect("Failed to insert timeline safekeepers"); - - assert_eq!(inserted, safekeeper_ids); - - // A second call with the same timeline should return the same safekeeper IDs. - let retrieved = idempotently_persist_or_get_existing_timeline_safekeepers( - &mut connection, - timeline1_id, - &[NodeId(4), NodeId(5), NodeId(6)], - ) - .await - .expect("Failed to retrieve timeline safekeepers"); - - assert_eq!(retrieved, safekeeper_ids); - } - - async fn load_timelines_by_sk_node( - conn: &mut AsyncPgConnection, - ) -> DatabaseResult>> { - use crate::schema::hadron_timeline_safekeepers; - - let rows = hadron_timeline_safekeepers::table - .select(( - hadron_timeline_safekeepers::sk_node_id, - hadron_timeline_safekeepers::timeline_id, - )) - .load::<(i64, String)>(conn) - .await?; - - let mut timelines_by_sk_node = BTreeMap::new(); - for (sk_node_id, timeline_id) in rows { - timelines_by_sk_node - .entry(sk_node_id) - .or_insert_with(Vec::new) - .push(timeline_id); - } - - Ok(timelines_by_sk_node) - } - - #[tokio::test] - async fn test_delete_timeline_safekeepers() { - let pg = get_embedded_pg().await.unwrap(); - - let connection_string = pg.settings().url("test"); - { - let mut conn = PgConnection::establish(&connection_string) - .unwrap_or_else(|_| panic!("Error connecting to {connection_string}")); - run_migrations(&mut conn).unwrap(); - } - - let mut connection = AsyncPgConnection::establish(&connection_string) - .await - .unwrap_or_else(|_| panic!("Error connecting to {connection_string}")); - - // Insert some values - let timeline1_id = TimelineId::generate(); - let safekeeper_ids = vec![NodeId(1), NodeId(2), NodeId(3)]; - idempotently_persist_or_get_existing_timeline_safekeepers( - &mut connection, - timeline1_id, - &safekeeper_ids, - ) - .await - .expect("Failed to insert timeline safekeepers"); - - // Validate that the values were inserted - let inserted = load_timelines_by_sk_node(&mut connection) - .await - .expect("Failed to load timelines by sk node"); - - assert_eq!(inserted.get(&1).unwrap().len(), 1); - assert_eq!(inserted.get(&2).unwrap().len(), 1); - assert_eq!(inserted.get(&3).unwrap().len(), 1); - - // Delete the values - delete_timeline_safekeepers(&mut connection, timeline1_id) - .await - .expect("Failed to delete timeline safekeepers"); - - // Validate that the values were deleted - let deleted = load_timelines_by_sk_node(&mut connection) - .await - .expect("Failed to load timelines by sk node"); - - assert!(deleted.is_empty()); - } - - fn assert_list_safekeeper_timelines( - actual: &mut SCSafekeeperTimelinesResponse, - expected: &mut SCSafekeeperTimelinesResponse, - ) { - assert_eq!(actual.timelines.len(), expected.timelines.len()); - assert_eq!( - actual.safekeeper_peers.len(), - expected.safekeeper_peers.len() - ); - - actual.timelines.sort_by_key(|item| item.timeline_id); - expected.timelines.sort_by_key(|item| item.timeline_id); - - actual.safekeeper_peers.sort_by_key(|item| item.node_id); - expected.safekeeper_peers.sort_by_key(|item| item.node_id); - - for i in 0..actual.timelines.len() { - let mut at = actual.timelines[i].clone(); - let mut et = expected.timelines[i].clone(); - at.peers.sort_by_key(|item| item.0); - et.peers.sort_by_key(|item| item.0); - - assert_eq!(at.timeline_id, et.timeline_id); - - assert!( - at.peers.iter().eq(et.peers.iter()), - "at peers: {:#?}, et peers: {:#?}", - at.peers, - et.peers - ); - } - - for i in 0..actual.safekeeper_peers.len() { - let at = actual.safekeeper_peers[i].clone(); - let et = expected.safekeeper_peers[i].clone(); - assert_eq!(at.node_id, et.node_id); - assert_eq!(at.listen_http_addr, et.listen_http_addr); - assert_eq!(at.http_port, et.http_port); - } - } - - #[tokio::test] - async fn test_list_safekeeper_timelines() { - let pg = get_embedded_pg().await.unwrap(); - - let connection_string = pg.settings().url("test"); - { - let mut conn = PgConnection::establish(&connection_string) - .unwrap_or_else(|_| panic!("Error connecting to {connection_string}")); - run_migrations(&mut conn).unwrap(); - } - let mut connection = AsyncPgConnection::establish(&connection_string) - .await - .unwrap_or_else(|_| panic!("Error connecting to {connection_string}")); - - // Insert some values - let safekeeper_ids = vec![ - NodeId(0), - NodeId(1), - NodeId(2), - NodeId(3), - NodeId(4), - NodeId(5), - ]; - for safekeeper_id in &safekeeper_ids { - execute_sk_upsert( - &mut connection, - get_test_sk_node(safekeeper_id.0).to_database_row(), - ) - .await - .unwrap(); - } - - // Create some endpoints. - // 5 use SK-0/1/2 - // 5 use SK-2/3/4 - let mut timeline_ids = Vec::new(); - - for i in 0..10 { - let timeline_id = TimelineId::generate(); - timeline_ids.push(timeline_id); - - let safekeepers = if i < 5 { - vec![NodeId(0), NodeId(1), NodeId(2)] - } else { - vec![NodeId(2), NodeId(3), NodeId(4)] - }; - - idempotently_persist_or_get_existing_timeline_safekeepers( - &mut connection, - timeline_id, - &safekeepers, - ) - .await - .unwrap(); - } - - // SK-0/1 owns the first 5 timelines. - // SK-2 owns all 10 timelines. - // SK-3/4 owns the last 5 timelines. - // SK-5 owns no timelines. - // SK-6 does not exist. - let mut expected_responses = vec![ - SCSafekeeperTimelinesResponse { - timelines: Vec::new(), - safekeeper_peers: Vec::new(), - }; - 7 - ]; - - // SC does not know the tenant ids. - for (i, timeline_id) in timeline_ids.iter().enumerate().take(10) { - if i < 5 { - expected_responses[0].timelines.push(SCSafekeeperTimeline { - timeline_id: *timeline_id, - peers: vec![NodeId(0), NodeId(1), NodeId(2)], - }); - - expected_responses[2].timelines.push(SCSafekeeperTimeline { - timeline_id: *timeline_id, - peers: vec![NodeId(0), NodeId(1), NodeId(2)], - }); - continue; - } - - expected_responses[2].timelines.push(SCSafekeeperTimeline { - timeline_id: *timeline_id, - peers: vec![NodeId(2), NodeId(3), NodeId(4)], - }); - expected_responses[3].timelines.push(SCSafekeeperTimeline { - timeline_id: *timeline_id, - peers: vec![NodeId(2), NodeId(3), NodeId(4)], - }); - } - for i in 0..5 { - expected_responses[2] - .safekeeper_peers - .push(TimelineSafekeeperPeer { - node_id: NodeId(i), - listen_http_addr: format!("safekeeper-{i}"), - http_port: 123, - }); - if i < 3 { - expected_responses[0] - .safekeeper_peers - .push(TimelineSafekeeperPeer { - node_id: NodeId(i), - listen_http_addr: format!("safekeeper-{i}"), - http_port: 123, - }); - expected_responses[3] - .safekeeper_peers - .push(TimelineSafekeeperPeer { - node_id: NodeId(i + 2), - listen_http_addr: format!("safekeeper-{}", i + 2), - http_port: 123, - }); - } - } - expected_responses[1] = expected_responses[0].clone(); - expected_responses[4] = expected_responses[3].clone(); - - for safekeeper_id in &safekeeper_ids { - let sk_timelines: Result = - execute_safekeeper_list_timelines( - &mut connection, - safekeeper_id.0.try_into().unwrap(), - ) - .await; - assert!(sk_timelines.is_ok()); - let mut sk_timelines: SCSafekeeperTimelinesResponse = sk_timelines.unwrap(); - assert_list_safekeeper_timelines( - &mut sk_timelines, - &mut expected_responses[safekeeper_id.0 as usize], - ); - } - } -} diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index c5561bb617..952cd9d3be 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -18,7 +18,6 @@ license.workspace = true ahash = { version = "0.8" } anstream = { version = "0.6" } anyhow = { version = "1", features = ["backtrace"] } -arrayvec = { version = "0.7" } axum = { version = "0.8", features = ["ws"] } axum-core = { version = "0.5", default-features = false, features = ["tracing"] } base64 = { version = "0.21" } @@ -28,21 +27,18 @@ camino = { version = "1", default-features = false, features = ["serde1"] } chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "wasmbind"] } clap = { version = "4", features = ["derive", "env", "string"] } clap_builder = { version = "4", default-features = false, features = ["color", "env", "help", "std", "string", "suggestions", "usage"] } -concurrent-queue = { version = "2" } const-oid = { version = "0.9", default-features = false, features = ["db", "std"] } crossbeam-epoch = { version = "0.9" } crossbeam-utils = { version = "0.8" } crypto-bigint = { version = "0.5", features = ["generic-array", "zeroize"] } der = { version = "0.7", default-features = false, features = ["derive", "flagset", "oid", "pem", "std"] } deranged = { version = "0.3", default-features = false, features = ["powerfmt", "serde", "std"] } -diesel = { version = "2", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "postgres", "serde_json", "uuid"] } digest = { version = "0.10", features = ["mac", "oid", "std"] } ecdsa = { version = "0.16", features = ["pem", "signing", "std", "verifying"] } -either = { version = "1", features = ["serde"] } +either = { version = "1" } elliptic-curve = { version = "0.13", default-features = false, features = ["digest", "hazmat", "jwk", "pem", "std"] } env_filter = { version = "0.1", default-features = false, features = ["regex"] } env_logger = { version = "0.11" } -event-listener = { version = "5" } fail = { version = "0.5", default-features = false, features = ["failpoints"] } form_urlencoded = { version = "1" } futures-channel = { version = "0.3", features = ["sink"] } @@ -54,8 +50,7 @@ futures-util = { version = "0.3", features = ["channel", "io", "sink"] } generic-array = { version = "0.14", default-features = false, features = ["more_lengths", "zeroize"] } getrandom = { version = "0.2", default-features = false, features = ["std"] } half = { version = "2", default-features = false, features = ["num-traits"] } -hashbrown-3575ec1268b04181 = { package = "hashbrown", version = "0.15" } -hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14", features = ["raw"] } +hashbrown = { version = "0.14", features = ["raw"] } hex = { version = "0.4", features = ["serde"] } hmac = { version = "0.12", default-features = false, features = ["reset"] } hyper-582f2526e08bb6a0 = { package = "hyper", version = "0.14", features = ["client", "http1", "http2", "runtime", "server", "stream"] } @@ -73,36 +68,30 @@ nom = { version = "7" } num = { version = "0.4" } num-bigint = { version = "0.4" } num-complex = { version = "0.4", default-features = false, features = ["std"] } -num-format = { version = "0.4" } num-integer = { version = "0.1", features = ["i128"] } num-iter = { version = "0.1", default-features = false, features = ["i128", "std"] } num-rational = { version = "0.4", default-features = false, features = ["num-bigint-std", "std"] } num-traits = { version = "0.2", features = ["i128", "libm"] } once_cell = { version = "1" } -opentelemetry_sdk = { version = "0.30", features = ["rt-tokio"] } p256 = { version = "0.13", features = ["jwk"] } parquet = { version = "53", default-features = false, features = ["zstd"] } -percent-encoding = { version = "2" } portable-atomic = { version = "1", features = ["require-cas"] } -postgresql_archive = { version = "0.19" } prost = { version = "0.13", features = ["no-recursion-limit", "prost-derive"] } rand = { version = "0.9" } regex = { version = "1" } regex-automata = { version = "0.4", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } regex-syntax = { version = "0.8" } -reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "native-tls", "rustls-tls", "rustls-tls-native-roots", "stream"] } -reqwest-middleware = { version = "0.4", default-features = false, features = ["json"] } -reqwest-tracing = { version = "0.5", default-features = false, features = ["opentelemetry_0_30"] } +reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls", "rustls-tls-native-roots", "stream"] } rustls = { version = "0.23", default-features = false, features = ["logging", "ring", "std", "tls12"] } rustls-pki-types = { version = "1", features = ["std"] } rustls-webpki = { version = "0.103", default-features = false, features = ["ring", "std"] } scopeguard = { version = "1" } sec1 = { version = "0.7", features = ["pem", "serde", "std", "subtle"] } -serde = { version = "1", features = ["alloc", "derive", "rc"] } +serde = { version = "1", features = ["alloc", "derive"] } serde_json = { version = "1", features = ["alloc", "raw_value"] } sha2 = { version = "0.10", features = ["asm", "oid"] } signature = { version = "2", default-features = false, features = ["digest", "rand_core", "std"] } -smallvec = { version = "1", default-features = false, features = ["const_new", "serde", "write"] } +smallvec = { version = "1", default-features = false, features = ["const_new", "write"] } spki = { version = "0.7", default-features = false, features = ["pem", "std"] } stable_deref_trait = { version = "1" } subtle = { version = "2" } @@ -113,7 +102,7 @@ tikv-jemalloc-sys = { version = "0.6", features = ["profiling", "stats", "unpref time = { version = "0.3", features = ["macros", "serde-well-known"] } tokio = { version = "1", features = ["full", "test-util"] } tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "ring", "tls12"] } -tokio-stream = { version = "0.1", features = ["fs", "net", "sync"] } +tokio-stream = { version = "0.1", features = ["net", "sync"] } tokio-util = { version = "0.7", features = ["codec", "compat", "io-util", "rt"] } toml_edit = { version = "0.22", features = ["serde"] } tonic = { version = "0.13", default-features = false, features = ["codegen", "gzip", "prost", "router", "server", "tls-native-roots", "tls-ring", "zstd"] } @@ -123,7 +112,6 @@ tracing-core = { version = "0.1" } tracing-log = { version = "0.2" } tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } url = { version = "2", features = ["serde"] } -uuid = { version = "1", features = ["serde", "v4", "v7"] } zeroize = { version = "1", features = ["derive", "serde"] } zstd = { version = "0.13" } zstd-safe = { version = "7", default-features = false, features = ["arrays", "legacy", "std", "zdict_builder"] } @@ -133,31 +121,17 @@ zstd-sys = { version = "2", default-features = false, features = ["legacy", "std ahash = { version = "0.8" } anstream = { version = "0.6" } anyhow = { version = "1", features = ["backtrace"] } -arrayvec = { version = "0.7" } bytes = { version = "1", features = ["serde"] } cc = { version = "1", default-features = false, features = ["parallel"] } chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "wasmbind"] } clap = { version = "4", features = ["derive", "env", "string"] } clap_builder = { version = "4", default-features = false, features = ["color", "env", "help", "std", "string", "suggestions", "usage"] } -const-oid = { version = "0.9", default-features = false, features = ["db", "std"] } -digest = { version = "0.10", features = ["mac", "oid", "std"] } -either = { version = "1", features = ["serde"] } -form_urlencoded = { version = "1" } -futures-channel = { version = "0.3", features = ["sink"] } -futures-core = { version = "0.3" } -futures-executor = { version = "0.3" } -futures-io = { version = "0.3" } -futures-sink = { version = "0.3" } -futures-util = { version = "0.3", features = ["channel", "io", "sink"] } -generic-array = { version = "0.14", default-features = false, features = ["more_lengths", "zeroize"] } +either = { version = "1" } getrandom = { version = "0.2", default-features = false, features = ["std"] } half = { version = "2", default-features = false, features = ["num-traits"] } -hashbrown-3575ec1268b04181 = { package = "hashbrown", version = "0.15" } -hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14", features = ["raw"] } -hex = { version = "0.4", features = ["serde"] } +hashbrown = { version = "0.14", features = ["raw"] } indexmap = { version = "2", features = ["serde"] } itertools = { version = "0.12" } -lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } libc = { version = "0.2", features = ["extra_traits", "use_std"] } log = { version = "0.4", default-features = false, features = ["std"] } memchr = { version = "2" } @@ -165,48 +139,24 @@ nom = { version = "7" } num = { version = "0.4" } num-bigint = { version = "0.4" } num-complex = { version = "0.4", default-features = false, features = ["std"] } -num-format = { version = "0.4" } num-integer = { version = "0.1", features = ["i128"] } num-iter = { version = "0.1", default-features = false, features = ["i128", "std"] } num-rational = { version = "0.4", default-features = false, features = ["num-bigint-std", "std"] } num-traits = { version = "0.2", features = ["i128", "libm"] } once_cell = { version = "1" } -opentelemetry_sdk = { version = "0.30", features = ["rt-tokio"] } parquet = { version = "53", default-features = false, features = ["zstd"] } -percent-encoding = { version = "2" } -postgresql_archive = { version = "0.19" } prettyplease = { version = "0.2", default-features = false, features = ["verbatim"] } proc-macro2 = { version = "1" } prost = { version = "0.13", features = ["no-recursion-limit", "prost-derive"] } quote = { version = "1" } -rand = { version = "0.9" } regex = { version = "1" } regex-automata = { version = "0.4", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } regex-syntax = { version = "0.8" } -reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "native-tls", "rustls-tls", "rustls-tls-native-roots", "stream"] } -reqwest-middleware = { version = "0.4", default-features = false, features = ["json"] } -reqwest-tracing = { version = "0.5", default-features = false, features = ["opentelemetry_0_30"] } -scopeguard = { version = "1" } -serde = { version = "1", features = ["alloc", "derive", "rc"] } +serde = { version = "1", features = ["alloc", "derive"] } serde_json = { version = "1", features = ["alloc", "raw_value"] } -sha2 = { version = "0.10", features = ["asm", "oid"] } -smallvec = { version = "1", default-features = false, features = ["const_new", "serde", "write"] } -stable_deref_trait = { version = "1" } -subtle = { version = "2" } syn = { version = "2", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } -sync_wrapper = { version = "1", default-features = false, features = ["futures"] } -thiserror = { version = "2" } time-macros = { version = "0.2", default-features = false, features = ["formatting", "parsing", "serde"] } -tokio = { version = "1", features = ["full", "test-util"] } -tokio-stream = { version = "0.1", features = ["fs", "net", "sync"] } -tokio-util = { version = "0.7", features = ["codec", "compat", "io-util", "rt"] } toml_edit = { version = "0.22", features = ["serde"] } -tracing = { version = "0.1", features = ["log"] } -tracing-core = { version = "0.1" } -tracing-log = { version = "0.2" } -tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } -url = { version = "2", features = ["serde"] } -zeroize = { version = "1", features = ["derive", "serde"] } zstd = { version = "0.13" } zstd-safe = { version = "7", default-features = false, features = ["arrays", "legacy", "std", "zdict_builder"] } zstd-sys = { version = "2", default-features = false, features = ["legacy", "std", "zdict_builder"] }