tests: remove tests that depend on libpq

This commit is contained in:
Vlad Lazar
2025-07-29 17:35:47 +01:00
parent e92ed85e9a
commit 2f9cc9a11e
4 changed files with 10 additions and 904 deletions

420
Cargo.lock generated
View File

@@ -247,15 +247,6 @@ dependencies = [
"syn 2.0.100",
]
[[package]]
name = "atoi"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528"
dependencies = [
"num-traits",
]
[[package]]
name = "atomic-take"
version = "1.1.0"
@@ -1550,21 +1541,6 @@ dependencies = [
"libc",
]
[[package]]
name = "crc"
version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675"
dependencies = [
"crc-catalog",
]
[[package]]
name = "crc-catalog"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
[[package]]
name = "crc32c"
version = "0.6.8"
@@ -1659,15 +1635,6 @@ dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-queue"
version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.19"
@@ -1915,7 +1882,6 @@ dependencies = [
"chrono",
"diesel_derives",
"itoa",
"pq-sys",
"serde_json",
"uuid",
]
@@ -2006,12 +1972,6 @@ dependencies = [
"const-random",
]
[[package]]
name = "dotenvy"
version = "0.15.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
[[package]]
name = "dsl_auto_type"
version = "0.1.1"
@@ -2085,9 +2045,6 @@ name = "either"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
dependencies = [
"serde",
]
[[package]]
name = "elliptic-curve"
@@ -2261,17 +2218,6 @@ dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "etcetera"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943"
dependencies = [
"cfg-if",
"home",
"windows-sys 0.48.0",
]
[[package]]
name = "event-listener"
version = "2.5.3"
@@ -2409,27 +2355,6 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foldhash"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "foreign-types"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
dependencies = [
"foreign-types-shared",
]
[[package]]
name = "foreign-types-shared"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]]
name = "form_urlencoded"
version = "1.2.1"
@@ -2511,17 +2436,6 @@ dependencies = [
"futures-util",
]
[[package]]
name = "futures-intrusive"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f"
dependencies = [
"futures-core",
"lock_api",
"parking_lot 0.12.1",
]
[[package]]
name = "futures-io"
version = "0.3.31"
@@ -2823,11 +2737,6 @@ name = "hashbrown"
version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash",
]
[[package]]
name = "hashlink"
@@ -2838,15 +2747,6 @@ dependencies = [
"hashbrown 0.14.5",
]
[[package]]
name = "hashlink"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
dependencies = [
"hashbrown 0.15.2",
]
[[package]]
name = "hdrhistogram"
version = "7.5.4"
@@ -2927,15 +2827,6 @@ dependencies = [
"never-say-never",
]
[[package]]
name = "hkdf"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7"
dependencies = [
"hmac",
]
[[package]]
name = "hmac"
version = "0.12.1"
@@ -2945,15 +2836,6 @@ dependencies = [
"digest",
]
[[package]]
name = "home"
version = "0.5.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf"
dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "hostname"
version = "0.4.0"
@@ -3203,22 +3085,6 @@ dependencies = [
"tower-service",
]
[[package]]
name = "hyper-tls"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
dependencies = [
"bytes",
"http-body-util",
"hyper 1.6.0",
"hyper-util",
"native-tls",
"tokio",
"tokio-native-tls",
"tower-service",
]
[[package]]
name = "hyper-util"
version = "0.1.16"
@@ -4084,23 +3950,6 @@ version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
[[package]]
name = "native-tls"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e"
dependencies = [
"libc",
"log",
"openssl",
"openssl-probe",
"openssl-sys",
"schannel",
"security-framework",
"security-framework-sys",
"tempfile",
]
[[package]]
name = "neon-shmem"
version = "0.1.0"
@@ -4378,50 +4227,12 @@ version = "11.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
[[package]]
name = "openssl"
version = "0.10.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8"
dependencies = [
"bitflags 2.8.0",
"cfg-if",
"foreign-types",
"libc",
"once_cell",
"openssl-macros",
"openssl-sys",
]
[[package]]
name = "openssl-macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.100",
]
[[package]]
name = "openssl-probe"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
[[package]]
name = "openssl-sys"
version = "0.9.109"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571"
dependencies = [
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "opentelemetry"
version = "0.30.0"
@@ -4681,7 +4492,7 @@ dependencies = [
"enumset",
"fail",
"futures",
"hashlink 0.9.1",
"hashlink",
"hex",
"hex-literal",
"http 1.3.1",
@@ -5336,65 +5147,6 @@ dependencies = [
"workspace_hack",
]
[[package]]
name = "postgresql_archive"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2b10a6a63e5f4ccf5d99d7d5db0829aec30a5e84b5c80883f6561c77a7521a6"
dependencies = [
"async-trait",
"flate2",
"futures-util",
"hex",
"num-format",
"regex-lite",
"reqwest",
"reqwest-middleware",
"reqwest-retry",
"reqwest-tracing",
"semver",
"serde",
"serde_json",
"sha2",
"tar",
"target-triple",
"tempfile",
"thiserror 2.0.12",
"tracing",
"url",
]
[[package]]
name = "postgresql_commands"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cc8653c31df5ffcd3bc114641dfae5390404d6e22bc21fd7473760f22cdc20b"
dependencies = [
"thiserror 2.0.12",
"tokio",
"tracing",
]
[[package]]
name = "postgresql_embedded"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4386829807c8dfbc190f8e53b91d2080f0f604bd4b9f0831281ccfbc06998123"
dependencies = [
"anyhow",
"postgresql_archive",
"postgresql_commands",
"rand 0.9.1",
"semver",
"sqlx",
"target-triple",
"tempfile",
"thiserror 2.0.12",
"tokio",
"tracing",
"url",
]
[[package]]
name = "posthog_client_lite"
version = "0.1.0"
@@ -5467,15 +5219,6 @@ version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
[[package]]
name = "pq-sys"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6cc05d7ea95200187117196eee9edd0644424911821aeb28a18ce60ea0b8793"
dependencies = [
"vcpkg",
]
[[package]]
name = "pq_proto"
version = "0.1.0"
@@ -6258,11 +6001,9 @@ dependencies = [
"http-body-util",
"hyper 1.6.0",
"hyper-rustls 0.27.7",
"hyper-tls",
"hyper-util",
"js-sys",
"log",
"native-tls",
"percent-encoding",
"pin-project-lite",
"quinn",
@@ -6274,7 +6015,6 @@ dependencies = [
"serde_urlencoded",
"sync_wrapper 1.0.1",
"tokio",
"tokio-native-tls",
"tokio-rustls 0.26.2",
"tokio-util",
"tower",
@@ -7255,9 +6995,6 @@ name = "smallvec"
version = "1.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
dependencies = [
"serde",
]
[[package]]
name = "smol_str"
@@ -7323,125 +7060,6 @@ dependencies = [
"der 0.7.8",
]
[[package]]
name = "sqlx"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc"
dependencies = [
"sqlx-core",
"sqlx-macros",
"sqlx-postgres",
]
[[package]]
name = "sqlx-core"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6"
dependencies = [
"base64 0.22.1",
"bytes",
"crc",
"crossbeam-queue",
"either",
"event-listener 5.4.0",
"futures-core",
"futures-intrusive",
"futures-io",
"futures-util",
"hashbrown 0.15.2",
"hashlink 0.10.0",
"indexmap 2.10.0",
"log",
"memchr",
"native-tls",
"once_cell",
"percent-encoding",
"serde",
"serde_json",
"sha2",
"smallvec",
"thiserror 2.0.12",
"tokio",
"tokio-stream",
"tracing",
"url",
]
[[package]]
name = "sqlx-macros"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d"
dependencies = [
"proc-macro2",
"quote",
"sqlx-core",
"sqlx-macros-core",
"syn 2.0.100",
]
[[package]]
name = "sqlx-macros-core"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b"
dependencies = [
"dotenvy",
"either",
"heck 0.5.0",
"hex",
"once_cell",
"proc-macro2",
"quote",
"serde",
"serde_json",
"sha2",
"sqlx-core",
"sqlx-postgres",
"syn 2.0.100",
"tokio",
"url",
]
[[package]]
name = "sqlx-postgres"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46"
dependencies = [
"atoi",
"base64 0.22.1",
"bitflags 2.8.0",
"byteorder",
"crc",
"dotenvy",
"etcetera",
"futures-channel",
"futures-core",
"futures-util",
"hex",
"hkdf",
"hmac",
"home",
"itoa",
"log",
"md-5",
"memchr",
"once_cell",
"rand 0.8.5",
"serde",
"serde_json",
"sha2",
"smallvec",
"sqlx-core",
"stringprep",
"thiserror 2.0.12",
"tracing",
"whoami",
]
[[package]]
name = "stable_deref_trait"
version = "1.2.0"
@@ -7519,8 +7137,6 @@ dependencies = [
"pageserver_api",
"pageserver_client",
"postgres_connection",
"postgresql_archive",
"postgresql_embedded",
"posthog_client_lite",
"rand 0.9.1",
"regex",
@@ -7787,12 +7403,6 @@ dependencies = [
"xattr",
]
[[package]]
name = "target-triple"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ac9aa371f599d22256307c24a9d748c041e548cbf599f35d890f9d365361790"
[[package]]
name = "tempfile"
version = "3.20.0"
@@ -8104,16 +7714,6 @@ dependencies = [
"syn 2.0.100",
]
[[package]]
name = "tokio-native-tls"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
dependencies = [
"native-tls",
"tokio",
]
[[package]]
name = "tokio-postgres"
version = "0.7.10"
@@ -8831,12 +8431,6 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "version_check"
version = "0.9.4"
@@ -9456,7 +9050,6 @@ dependencies = [
"ahash",
"anstream",
"anyhow",
"arrayvec",
"axum",
"axum-core",
"base64 0.21.7",
@@ -9467,21 +9060,18 @@ dependencies = [
"chrono",
"clap",
"clap_builder",
"concurrent-queue",
"const-oid",
"crossbeam-epoch",
"crossbeam-utils",
"crypto-bigint 0.5.5",
"der 0.7.8",
"deranged",
"diesel",
"digest",
"ecdsa 0.16.9",
"either",
"elliptic-curve 0.13.8",
"env_filter",
"env_logger",
"event-listener 5.4.0",
"fail",
"form_urlencoded",
"futures-channel",
@@ -9494,7 +9084,6 @@ dependencies = [
"getrandom 0.2.11",
"half",
"hashbrown 0.14.5",
"hashbrown 0.15.2",
"hex",
"hmac",
"hyper 0.14.30",
@@ -9512,18 +9101,14 @@ dependencies = [
"num",
"num-bigint",
"num-complex",
"num-format",
"num-integer",
"num-iter",
"num-rational",
"num-traits",
"once_cell",
"opentelemetry_sdk",
"p256 0.13.2",
"parquet",
"percent-encoding",
"portable-atomic",
"postgresql_archive",
"prettyplease",
"proc-macro2",
"prost 0.13.5",
@@ -9533,8 +9118,6 @@ dependencies = [
"regex-automata 0.4.9",
"regex-syntax 0.8.5",
"reqwest",
"reqwest-middleware",
"reqwest-tracing",
"rustls 0.23.29",
"rustls-pki-types",
"rustls-webpki 0.103.4",
@@ -9567,7 +9150,6 @@ dependencies = [
"tracing-log",
"tracing-subscriber",
"url",
"uuid",
"zeroize",
"zstd",
"zstd-safe",

View File

@@ -75,13 +75,3 @@ utils = { path = "../libs/utils/" }
metrics = { path = "../libs/metrics/" }
control_plane = { path = "../control_plane" }
workspace_hack = { version = "0.1", path = "../workspace_hack" }
[dev-dependencies]
postgresql_archive = "0.19.0"
postgresql_embedded = { version = "0.19.0", features = ["blocking"] }
diesel = { version = "2.2.6", features = [
"postgres",
"serde_json",
"chrono",
"uuid",
] }

View File

@@ -431,419 +431,3 @@ async fn get_pageserver_connection_info(
}
Ok(shards)
}
#[cfg(test)]
mod test {
use std::collections::BTreeMap;
use super::*;
use crate::schema::hadron_safekeepers;
use diesel::PgConnection;
use diesel_migrations::{EmbeddedMigrations, MigrationHarness, embed_migrations};
use pageserver_api::controller_api::{SCSafekeeperTimeline, TimelineSafekeeperPeer};
use postgresql_archive::VersionReq;
use postgresql_embedded::Settings;
use postgresql_embedded::blocking::PostgreSQL;
async fn get_embedded_pg() -> postgresql_embedded::Result<PostgreSQL> {
tokio::task::spawn_blocking(|| {
let pg_install_dir = "../pg_install/16.0.0";
// Link "pg_install/v16" -> "pg_install/16.0.0" so that it can be picked up by the postgres_embedded
// crate without needing to download anything. The postgres_embedded crate expects a specific format
// for the directory name.
let _ = std::os::unix::fs::symlink("./v16", pg_install_dir);
let settings = Settings {
installation_dir: std::path::PathBuf::from(pg_install_dir),
username: "postgres".to_string(),
password: "password".to_string(),
// Use a 30-second timeout for database initialization to avoid flakiness in the CI environment.
timeout: Some(std::time::Duration::from_secs(30)),
version: VersionReq::parse("=16.0.0").unwrap(),
..Default::default()
};
let mut pg = PostgreSQL::new(settings);
pg.setup()?;
pg.start()?;
pg.create_database("test")?;
Ok(pg)
})
.await
.unwrap()
}
pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations");
fn run_migrations(connection: &mut PgConnection) -> Result<(), String> {
connection.run_pending_migrations(MIGRATIONS).unwrap();
Ok(())
}
fn get_test_sk_node(id: u64) -> SafeKeeperNode {
SafeKeeperNode::new(
NodeId(id),
format!("safekeeper-{id}"),
123,
format!("safekeeper-{id}"),
456,
)
}
#[tokio::test]
async fn test_safekeeper_upserts_and_list() {
let pg = get_embedded_pg().await.unwrap();
let connection_string = pg.settings().url("test");
{
let mut conn = PgConnection::establish(&connection_string)
.unwrap_or_else(|_| panic!("Error connecting to {connection_string}"));
run_migrations(&mut conn).unwrap();
}
let mut connection = AsyncPgConnection::establish(&connection_string)
.await
.unwrap_or_else(|_| panic!("Error connecting to {connection_string}"));
execute_sk_upsert(&mut connection, get_test_sk_node(0).to_database_row())
.await
.unwrap();
execute_sk_upsert(&mut connection, get_test_sk_node(1).to_database_row())
.await
.unwrap();
execute_sk_upsert(&mut connection, get_test_sk_node(2).to_database_row())
.await
.unwrap();
// Insert an entry into the hadron_timeline_safekeepers table.
use crate::schema::hadron_timeline_safekeepers;
let timeline1_id = TimelineId::generate();
diesel::insert_into(hadron_timeline_safekeepers::table)
.values(&HadronTimelineSafekeeper {
timeline_id: timeline1_id.to_string(),
sk_node_id: 0,
legacy_endpoint_id: None,
})
.execute(&mut connection)
.await
.expect("Failed to insert timeline1");
// Test that the nodes have indeed been inserted
let sk_nodes = hadron_safekeepers::table
.load::<HadronSafekeeperRow>(&mut connection)
.await
.unwrap();
assert_eq!(sk_nodes.len(), 3);
assert_eq!(sk_nodes[0].sk_node_id, 0);
assert_eq!(sk_nodes[1].sk_node_id, 1);
assert_eq!(sk_nodes[2].sk_node_id, 2);
// Test that we can read the nodes back out in the join query, where we pull all the Safekeepers along with their endpoints scheduled.
// There should be no endpoints in this test, verify that nothing breaks.
let sk_nodes = scan_safekeepers_and_scheduled_timelines(&mut connection)
.await
.unwrap();
assert_eq!(sk_nodes.len(), 3);
assert_eq!(sk_nodes[&NodeId(0)].legacy_endpoints.len(), 0);
assert_eq!(sk_nodes[&NodeId(1)].legacy_endpoints.len(), 0);
assert_eq!(sk_nodes[&NodeId(2)].legacy_endpoints.len(), 0);
// Test that only the 0th safekeeper is assigned to the timeline.
assert_eq!(sk_nodes[&NodeId(0)].timelines.len(), 1);
assert_eq!(sk_nodes[&NodeId(1)].timelines.len(), 0);
assert_eq!(sk_nodes[&NodeId(2)].timelines.len(), 0);
}
#[tokio::test]
async fn test_idempotently_persist_or_get_existing_timeline_safekeepers() {
let pg = get_embedded_pg().await.unwrap();
let connection_string = pg.settings().url("test");
{
let mut conn = PgConnection::establish(&connection_string)
.unwrap_or_else(|_| panic!("Error connecting to {connection_string}"));
run_migrations(&mut conn).unwrap();
}
let mut connection = AsyncPgConnection::establish(&connection_string)
.await
.unwrap_or_else(|_| panic!("Error connecting to {connection_string}"));
// An initial call should insert the timeline safekeepers and return the inserted values.
let timeline1_id = TimelineId::generate();
let safekeeper_ids = vec![NodeId(1), NodeId(2), NodeId(3)];
let inserted = idempotently_persist_or_get_existing_timeline_safekeepers(
&mut connection,
timeline1_id,
&safekeeper_ids,
)
.await
.expect("Failed to insert timeline safekeepers");
assert_eq!(inserted, safekeeper_ids);
// A second call with the same timeline should return the same safekeeper IDs.
let retrieved = idempotently_persist_or_get_existing_timeline_safekeepers(
&mut connection,
timeline1_id,
&[NodeId(4), NodeId(5), NodeId(6)],
)
.await
.expect("Failed to retrieve timeline safekeepers");
assert_eq!(retrieved, safekeeper_ids);
}
async fn load_timelines_by_sk_node(
conn: &mut AsyncPgConnection,
) -> DatabaseResult<BTreeMap<i64, Vec<String>>> {
use crate::schema::hadron_timeline_safekeepers;
let rows = hadron_timeline_safekeepers::table
.select((
hadron_timeline_safekeepers::sk_node_id,
hadron_timeline_safekeepers::timeline_id,
))
.load::<(i64, String)>(conn)
.await?;
let mut timelines_by_sk_node = BTreeMap::new();
for (sk_node_id, timeline_id) in rows {
timelines_by_sk_node
.entry(sk_node_id)
.or_insert_with(Vec::new)
.push(timeline_id);
}
Ok(timelines_by_sk_node)
}
#[tokio::test]
async fn test_delete_timeline_safekeepers() {
let pg = get_embedded_pg().await.unwrap();
let connection_string = pg.settings().url("test");
{
let mut conn = PgConnection::establish(&connection_string)
.unwrap_or_else(|_| panic!("Error connecting to {connection_string}"));
run_migrations(&mut conn).unwrap();
}
let mut connection = AsyncPgConnection::establish(&connection_string)
.await
.unwrap_or_else(|_| panic!("Error connecting to {connection_string}"));
// Insert some values
let timeline1_id = TimelineId::generate();
let safekeeper_ids = vec![NodeId(1), NodeId(2), NodeId(3)];
idempotently_persist_or_get_existing_timeline_safekeepers(
&mut connection,
timeline1_id,
&safekeeper_ids,
)
.await
.expect("Failed to insert timeline safekeepers");
// Validate that the values were inserted
let inserted = load_timelines_by_sk_node(&mut connection)
.await
.expect("Failed to load timelines by sk node");
assert_eq!(inserted.get(&1).unwrap().len(), 1);
assert_eq!(inserted.get(&2).unwrap().len(), 1);
assert_eq!(inserted.get(&3).unwrap().len(), 1);
// Delete the values
delete_timeline_safekeepers(&mut connection, timeline1_id)
.await
.expect("Failed to delete timeline safekeepers");
// Validate that the values were deleted
let deleted = load_timelines_by_sk_node(&mut connection)
.await
.expect("Failed to load timelines by sk node");
assert!(deleted.is_empty());
}
fn assert_list_safekeeper_timelines(
actual: &mut SCSafekeeperTimelinesResponse,
expected: &mut SCSafekeeperTimelinesResponse,
) {
assert_eq!(actual.timelines.len(), expected.timelines.len());
assert_eq!(
actual.safekeeper_peers.len(),
expected.safekeeper_peers.len()
);
actual.timelines.sort_by_key(|item| item.timeline_id);
expected.timelines.sort_by_key(|item| item.timeline_id);
actual.safekeeper_peers.sort_by_key(|item| item.node_id);
expected.safekeeper_peers.sort_by_key(|item| item.node_id);
for i in 0..actual.timelines.len() {
let mut at = actual.timelines[i].clone();
let mut et = expected.timelines[i].clone();
at.peers.sort_by_key(|item| item.0);
et.peers.sort_by_key(|item| item.0);
assert_eq!(at.timeline_id, et.timeline_id);
assert!(
at.peers.iter().eq(et.peers.iter()),
"at peers: {:#?}, et peers: {:#?}",
at.peers,
et.peers
);
}
for i in 0..actual.safekeeper_peers.len() {
let at = actual.safekeeper_peers[i].clone();
let et = expected.safekeeper_peers[i].clone();
assert_eq!(at.node_id, et.node_id);
assert_eq!(at.listen_http_addr, et.listen_http_addr);
assert_eq!(at.http_port, et.http_port);
}
}
#[tokio::test]
async fn test_list_safekeeper_timelines() {
let pg = get_embedded_pg().await.unwrap();
let connection_string = pg.settings().url("test");
{
let mut conn = PgConnection::establish(&connection_string)
.unwrap_or_else(|_| panic!("Error connecting to {connection_string}"));
run_migrations(&mut conn).unwrap();
}
let mut connection = AsyncPgConnection::establish(&connection_string)
.await
.unwrap_or_else(|_| panic!("Error connecting to {connection_string}"));
// Insert some values
let safekeeper_ids = vec![
NodeId(0),
NodeId(1),
NodeId(2),
NodeId(3),
NodeId(4),
NodeId(5),
];
for safekeeper_id in &safekeeper_ids {
execute_sk_upsert(
&mut connection,
get_test_sk_node(safekeeper_id.0).to_database_row(),
)
.await
.unwrap();
}
// Create some endpoints.
// 5 use SK-0/1/2
// 5 use SK-2/3/4
let mut timeline_ids = Vec::new();
for i in 0..10 {
let timeline_id = TimelineId::generate();
timeline_ids.push(timeline_id);
let safekeepers = if i < 5 {
vec![NodeId(0), NodeId(1), NodeId(2)]
} else {
vec![NodeId(2), NodeId(3), NodeId(4)]
};
idempotently_persist_or_get_existing_timeline_safekeepers(
&mut connection,
timeline_id,
&safekeepers,
)
.await
.unwrap();
}
// SK-0/1 owns the first 5 timelines.
// SK-2 owns all 10 timelines.
// SK-3/4 owns the last 5 timelines.
// SK-5 owns no timelines.
// SK-6 does not exist.
let mut expected_responses = vec![
SCSafekeeperTimelinesResponse {
timelines: Vec::new(),
safekeeper_peers: Vec::new(),
};
7
];
// SC does not know the tenant ids.
for (i, timeline_id) in timeline_ids.iter().enumerate().take(10) {
if i < 5 {
expected_responses[0].timelines.push(SCSafekeeperTimeline {
timeline_id: *timeline_id,
peers: vec![NodeId(0), NodeId(1), NodeId(2)],
});
expected_responses[2].timelines.push(SCSafekeeperTimeline {
timeline_id: *timeline_id,
peers: vec![NodeId(0), NodeId(1), NodeId(2)],
});
continue;
}
expected_responses[2].timelines.push(SCSafekeeperTimeline {
timeline_id: *timeline_id,
peers: vec![NodeId(2), NodeId(3), NodeId(4)],
});
expected_responses[3].timelines.push(SCSafekeeperTimeline {
timeline_id: *timeline_id,
peers: vec![NodeId(2), NodeId(3), NodeId(4)],
});
}
for i in 0..5 {
expected_responses[2]
.safekeeper_peers
.push(TimelineSafekeeperPeer {
node_id: NodeId(i),
listen_http_addr: format!("safekeeper-{i}"),
http_port: 123,
});
if i < 3 {
expected_responses[0]
.safekeeper_peers
.push(TimelineSafekeeperPeer {
node_id: NodeId(i),
listen_http_addr: format!("safekeeper-{i}"),
http_port: 123,
});
expected_responses[3]
.safekeeper_peers
.push(TimelineSafekeeperPeer {
node_id: NodeId(i + 2),
listen_http_addr: format!("safekeeper-{}", i + 2),
http_port: 123,
});
}
}
expected_responses[1] = expected_responses[0].clone();
expected_responses[4] = expected_responses[3].clone();
for safekeeper_id in &safekeeper_ids {
let sk_timelines: Result<SCSafekeeperTimelinesResponse, DatabaseError> =
execute_safekeeper_list_timelines(
&mut connection,
safekeeper_id.0.try_into().unwrap(),
)
.await;
assert!(sk_timelines.is_ok());
let mut sk_timelines: SCSafekeeperTimelinesResponse = sk_timelines.unwrap();
assert_list_safekeeper_timelines(
&mut sk_timelines,
&mut expected_responses[safekeeper_id.0 as usize],
);
}
}
}

View File

@@ -18,7 +18,6 @@ license.workspace = true
ahash = { version = "0.8" }
anstream = { version = "0.6" }
anyhow = { version = "1", features = ["backtrace"] }
arrayvec = { version = "0.7" }
axum = { version = "0.8", features = ["ws"] }
axum-core = { version = "0.5", default-features = false, features = ["tracing"] }
base64 = { version = "0.21" }
@@ -28,21 +27,18 @@ camino = { version = "1", default-features = false, features = ["serde1"] }
chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "wasmbind"] }
clap = { version = "4", features = ["derive", "env", "string"] }
clap_builder = { version = "4", default-features = false, features = ["color", "env", "help", "std", "string", "suggestions", "usage"] }
concurrent-queue = { version = "2" }
const-oid = { version = "0.9", default-features = false, features = ["db", "std"] }
crossbeam-epoch = { version = "0.9" }
crossbeam-utils = { version = "0.8" }
crypto-bigint = { version = "0.5", features = ["generic-array", "zeroize"] }
der = { version = "0.7", default-features = false, features = ["derive", "flagset", "oid", "pem", "std"] }
deranged = { version = "0.3", default-features = false, features = ["powerfmt", "serde", "std"] }
diesel = { version = "2", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "postgres", "serde_json", "uuid"] }
digest = { version = "0.10", features = ["mac", "oid", "std"] }
ecdsa = { version = "0.16", features = ["pem", "signing", "std", "verifying"] }
either = { version = "1", features = ["serde"] }
either = { version = "1" }
elliptic-curve = { version = "0.13", default-features = false, features = ["digest", "hazmat", "jwk", "pem", "std"] }
env_filter = { version = "0.1", default-features = false, features = ["regex"] }
env_logger = { version = "0.11" }
event-listener = { version = "5" }
fail = { version = "0.5", default-features = false, features = ["failpoints"] }
form_urlencoded = { version = "1" }
futures-channel = { version = "0.3", features = ["sink"] }
@@ -54,8 +50,7 @@ futures-util = { version = "0.3", features = ["channel", "io", "sink"] }
generic-array = { version = "0.14", default-features = false, features = ["more_lengths", "zeroize"] }
getrandom = { version = "0.2", default-features = false, features = ["std"] }
half = { version = "2", default-features = false, features = ["num-traits"] }
hashbrown-3575ec1268b04181 = { package = "hashbrown", version = "0.15" }
hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14", features = ["raw"] }
hashbrown = { version = "0.14", features = ["raw"] }
hex = { version = "0.4", features = ["serde"] }
hmac = { version = "0.12", default-features = false, features = ["reset"] }
hyper-582f2526e08bb6a0 = { package = "hyper", version = "0.14", features = ["client", "http1", "http2", "runtime", "server", "stream"] }
@@ -73,36 +68,30 @@ nom = { version = "7" }
num = { version = "0.4" }
num-bigint = { version = "0.4" }
num-complex = { version = "0.4", default-features = false, features = ["std"] }
num-format = { version = "0.4" }
num-integer = { version = "0.1", features = ["i128"] }
num-iter = { version = "0.1", default-features = false, features = ["i128", "std"] }
num-rational = { version = "0.4", default-features = false, features = ["num-bigint-std", "std"] }
num-traits = { version = "0.2", features = ["i128", "libm"] }
once_cell = { version = "1" }
opentelemetry_sdk = { version = "0.30", features = ["rt-tokio"] }
p256 = { version = "0.13", features = ["jwk"] }
parquet = { version = "53", default-features = false, features = ["zstd"] }
percent-encoding = { version = "2" }
portable-atomic = { version = "1", features = ["require-cas"] }
postgresql_archive = { version = "0.19" }
prost = { version = "0.13", features = ["no-recursion-limit", "prost-derive"] }
rand = { version = "0.9" }
regex = { version = "1" }
regex-automata = { version = "0.4", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] }
regex-syntax = { version = "0.8" }
reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "native-tls", "rustls-tls", "rustls-tls-native-roots", "stream"] }
reqwest-middleware = { version = "0.4", default-features = false, features = ["json"] }
reqwest-tracing = { version = "0.5", default-features = false, features = ["opentelemetry_0_30"] }
reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls", "rustls-tls-native-roots", "stream"] }
rustls = { version = "0.23", default-features = false, features = ["logging", "ring", "std", "tls12"] }
rustls-pki-types = { version = "1", features = ["std"] }
rustls-webpki = { version = "0.103", default-features = false, features = ["ring", "std"] }
scopeguard = { version = "1" }
sec1 = { version = "0.7", features = ["pem", "serde", "std", "subtle"] }
serde = { version = "1", features = ["alloc", "derive", "rc"] }
serde = { version = "1", features = ["alloc", "derive"] }
serde_json = { version = "1", features = ["alloc", "raw_value"] }
sha2 = { version = "0.10", features = ["asm", "oid"] }
signature = { version = "2", default-features = false, features = ["digest", "rand_core", "std"] }
smallvec = { version = "1", default-features = false, features = ["const_new", "serde", "write"] }
smallvec = { version = "1", default-features = false, features = ["const_new", "write"] }
spki = { version = "0.7", default-features = false, features = ["pem", "std"] }
stable_deref_trait = { version = "1" }
subtle = { version = "2" }
@@ -113,7 +102,7 @@ tikv-jemalloc-sys = { version = "0.6", features = ["profiling", "stats", "unpref
time = { version = "0.3", features = ["macros", "serde-well-known"] }
tokio = { version = "1", features = ["full", "test-util"] }
tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "ring", "tls12"] }
tokio-stream = { version = "0.1", features = ["fs", "net", "sync"] }
tokio-stream = { version = "0.1", features = ["net", "sync"] }
tokio-util = { version = "0.7", features = ["codec", "compat", "io-util", "rt"] }
toml_edit = { version = "0.22", features = ["serde"] }
tonic = { version = "0.13", default-features = false, features = ["codegen", "gzip", "prost", "router", "server", "tls-native-roots", "tls-ring", "zstd"] }
@@ -123,7 +112,6 @@ tracing-core = { version = "0.1" }
tracing-log = { version = "0.2" }
tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
url = { version = "2", features = ["serde"] }
uuid = { version = "1", features = ["serde", "v4", "v7"] }
zeroize = { version = "1", features = ["derive", "serde"] }
zstd = { version = "0.13" }
zstd-safe = { version = "7", default-features = false, features = ["arrays", "legacy", "std", "zdict_builder"] }
@@ -133,31 +121,17 @@ zstd-sys = { version = "2", default-features = false, features = ["legacy", "std
ahash = { version = "0.8" }
anstream = { version = "0.6" }
anyhow = { version = "1", features = ["backtrace"] }
arrayvec = { version = "0.7" }
bytes = { version = "1", features = ["serde"] }
cc = { version = "1", default-features = false, features = ["parallel"] }
chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "wasmbind"] }
clap = { version = "4", features = ["derive", "env", "string"] }
clap_builder = { version = "4", default-features = false, features = ["color", "env", "help", "std", "string", "suggestions", "usage"] }
const-oid = { version = "0.9", default-features = false, features = ["db", "std"] }
digest = { version = "0.10", features = ["mac", "oid", "std"] }
either = { version = "1", features = ["serde"] }
form_urlencoded = { version = "1" }
futures-channel = { version = "0.3", features = ["sink"] }
futures-core = { version = "0.3" }
futures-executor = { version = "0.3" }
futures-io = { version = "0.3" }
futures-sink = { version = "0.3" }
futures-util = { version = "0.3", features = ["channel", "io", "sink"] }
generic-array = { version = "0.14", default-features = false, features = ["more_lengths", "zeroize"] }
either = { version = "1" }
getrandom = { version = "0.2", default-features = false, features = ["std"] }
half = { version = "2", default-features = false, features = ["num-traits"] }
hashbrown-3575ec1268b04181 = { package = "hashbrown", version = "0.15" }
hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14", features = ["raw"] }
hex = { version = "0.4", features = ["serde"] }
hashbrown = { version = "0.14", features = ["raw"] }
indexmap = { version = "2", features = ["serde"] }
itertools = { version = "0.12" }
lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] }
libc = { version = "0.2", features = ["extra_traits", "use_std"] }
log = { version = "0.4", default-features = false, features = ["std"] }
memchr = { version = "2" }
@@ -165,48 +139,24 @@ nom = { version = "7" }
num = { version = "0.4" }
num-bigint = { version = "0.4" }
num-complex = { version = "0.4", default-features = false, features = ["std"] }
num-format = { version = "0.4" }
num-integer = { version = "0.1", features = ["i128"] }
num-iter = { version = "0.1", default-features = false, features = ["i128", "std"] }
num-rational = { version = "0.4", default-features = false, features = ["num-bigint-std", "std"] }
num-traits = { version = "0.2", features = ["i128", "libm"] }
once_cell = { version = "1" }
opentelemetry_sdk = { version = "0.30", features = ["rt-tokio"] }
parquet = { version = "53", default-features = false, features = ["zstd"] }
percent-encoding = { version = "2" }
postgresql_archive = { version = "0.19" }
prettyplease = { version = "0.2", default-features = false, features = ["verbatim"] }
proc-macro2 = { version = "1" }
prost = { version = "0.13", features = ["no-recursion-limit", "prost-derive"] }
quote = { version = "1" }
rand = { version = "0.9" }
regex = { version = "1" }
regex-automata = { version = "0.4", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] }
regex-syntax = { version = "0.8" }
reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "native-tls", "rustls-tls", "rustls-tls-native-roots", "stream"] }
reqwest-middleware = { version = "0.4", default-features = false, features = ["json"] }
reqwest-tracing = { version = "0.5", default-features = false, features = ["opentelemetry_0_30"] }
scopeguard = { version = "1" }
serde = { version = "1", features = ["alloc", "derive", "rc"] }
serde = { version = "1", features = ["alloc", "derive"] }
serde_json = { version = "1", features = ["alloc", "raw_value"] }
sha2 = { version = "0.10", features = ["asm", "oid"] }
smallvec = { version = "1", default-features = false, features = ["const_new", "serde", "write"] }
stable_deref_trait = { version = "1" }
subtle = { version = "2" }
syn = { version = "2", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
sync_wrapper = { version = "1", default-features = false, features = ["futures"] }
thiserror = { version = "2" }
time-macros = { version = "0.2", default-features = false, features = ["formatting", "parsing", "serde"] }
tokio = { version = "1", features = ["full", "test-util"] }
tokio-stream = { version = "0.1", features = ["fs", "net", "sync"] }
tokio-util = { version = "0.7", features = ["codec", "compat", "io-util", "rt"] }
toml_edit = { version = "0.22", features = ["serde"] }
tracing = { version = "0.1", features = ["log"] }
tracing-core = { version = "0.1" }
tracing-log = { version = "0.2" }
tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
url = { version = "2", features = ["serde"] }
zeroize = { version = "1", features = ["derive", "serde"] }
zstd = { version = "0.13" }
zstd-safe = { version = "7", default-features = false, features = ["arrays", "legacy", "std", "zdict_builder"] }
zstd-sys = { version = "2", default-features = false, features = ["legacy", "std", "zdict_builder"] }