starred/EasyTier
1
0
Fork 0
mirror of https://github.com/EasyTier/EasyTier.git synced 2026-10-13 00:00:34 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
1649ec00135e12103e05bf543e0ed313d7e190f8
EasyTier/easytier/src/tests/credential_tests.rs
T

3023 lines
92 KiB
Rust
Raw Blame History

//! Credential system integration tests
//!
//! These tests verify the credential-based authentication system where:
//! - Admin nodes hold network_secret and can generate credentials
//! - Credential nodes use X25519 keypairs to authenticate without network_secret
//! - Credentials can be revoked and propagate across the network
use std::time::Duration;
use easytier_core::{
foundation::stats::{LabelSet, LabelType, MetricName},
peers::credential_manager::CredentialCreateOptions,
process_runtime::CoreProcessRuntime,
};
#[cfg(feature = "wireguard")]
use crate::{
common::config::{VpnPortalClientConfig, VpnPortalConfig},
tests::three_node::{run_wireguard_client, wireguard_ifname},
vpn_portal::wireguard::test_wireguard_keys,
};
use crate::{
common::{
config::{ConfigLoader, NetworkIdentity, PeerConfig, TomlConfigLoader},
global_ctx::GlobalCtxEvent,
},
instance::test_instance::TestInstance as Instance,
proto::api::config::InstanceConfigPatch,
tests::three_node::{generate_secure_mode_config, generate_secure_mode_config_with_key},
tunnel::common::tests::wait_for_condition,
};
#[cfg(feature = "wireguard")]
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
#[cfg(feature = "wireguard")]
use defguard_wireguard_rs::key::Key;
#[cfg(feature = "wireguard")]
use easytier_core::gateway::vpn_portal::PortalClientState;
use super::{InstanceTestExt as _, drop_insts, network::TestNetwork, ping_test};
use rstest::rstest;
const PUBLIC_SERVER_NETWORK_NAME: &str = "__public_server__";
const PUBLIC_SERVER_SHARED_SECRET: &str = "public-server-shared-secret";
const NEED_P2P_ADMIN_NETWORK_NAME: &str = "need_p2p_credential_test_network";
async fn generate_credential(
admin: &Instance,
groups: Vec<String>,
allow_relay: bool,
allowed_proxy_cidrs: Vec<String>,
ttl: Duration,
) -> (String, String) {
generate_credential_with_options(
admin,
groups,
allow_relay,
allowed_proxy_cidrs,
ttl,
None,
true,
)
.await
}
async fn generate_credential_with_options(
admin: &Instance,
groups: Vec<String>,
allow_relay: bool,
allowed_proxy_cidrs: Vec<String>,
ttl: Duration,
credential_id: Option<String>,
reusable: bool,
) -> (String, String) {
let generated = admin
.get_core_instance()
.generate_credential(CredentialCreateOptions {
groups,
allow_relay,
allowed_proxy_cidrs,
ttl,
credential_id,
reusable,
})
.unwrap();
(generated.credential_id, generated.secret)
}
fn disable_p2p(config: &TomlConfigLoader) {
let mut flags = config.get_flags();
flags.disable_p2p = true;
config.set_flags(flags);
}
async fn set_prefer_peer_relay(inst: &Instance, prefer_peer_relay: bool) {
inst.get_config_patcher()
.apply_patch(InstanceConfigPatch {
prefer_peer_relay: Some(prefer_peer_relay),
..Default::default()
})
.await
.unwrap();
assert_eq!(
inst.get_global_ctx().get_flags().prefer_peer_relay,
prefer_peer_relay
);
}
fn forwarded_data_packets_for_network(inst: &Instance, network_name: &str) -> u64 {
let labels = LabelSet::new().with_label_type(LabelType::NetworkName(network_name.to_owned()));
inst.get_core_instance()
.metric_snapshots()
.into_iter()
.find(|metric| {
metric.name == MetricName::TrafficPacketsForwarded && metric.labels == labels
})
.map_or(0, |metric| metric.value)
}
fn forwarded_data_packets(inst: &Instance) -> u64 {
forwarded_data_packets_for_network(
inst,
&inst.get_global_ctx().get_network_identity().network_name,
)
}
async fn assert_ping_forwarded_by(
src_ns: &str,
dst_ip: &str,
expected_forwarder: &Instance,
forbidden_forwarders: &[&Instance],
) {
wait_ping_reachability(src_ns, dst_ip, true, Duration::from_secs(10)).await;
let expected_before = forwarded_data_packets(expected_forwarder);
let forbidden_before: Vec<_> = forbidden_forwarders
.iter()
.map(|inst| forwarded_data_packets(inst))
.collect();
for _ in 0..3 {
assert!(ping_test(src_ns, dst_ip, None).await);
}
wait_for_condition(
|| async { forwarded_data_packets(expected_forwarder) > expected_before },
Duration::from_secs(5),
)
.await;
for (forbidden_forwarder, forwarded_before) in forbidden_forwarders.iter().zip(forbidden_before)
{
assert_eq!(
forwarded_data_packets(forbidden_forwarder),
forwarded_before,
"data packets unexpectedly traversed a forbidden forwarder"
);
}
}
async fn wait_route_next_hop(
inst: &Instance,
peer_id: u32,
next_hop_peer_id: u32,
timeout: Duration,
label: &str,
) {
wait_for_condition(
|| async {
let routes = inst.get_core_instance().route_snapshots().await;
let next_hop = routes
.iter()
.find(|route| route.peer_id == peer_id)
.map(|route| route.next_hop_peer_id);
println!(
"{label}: next_hop={next_hop:?}, target={peer_id}, expected={next_hop_peer_id}"
);
next_hop == Some(next_hop_peer_id)
},
timeout,
)
.await;
}
async fn disconnect_connector(inst: &Instance, url: &url::Url, peer_id: u32) {
assert!(inst.get_core_instance().remove_connector(url));
let conn_ids = inst
.get_core_instance()
.peer_snapshots()
.await
.into_iter()
.find(|peer| peer.peer_id == peer_id)
.map(|peer| peer.directly_connected_conns)
.unwrap_or_default();
assert!(
!conn_ids.is_empty(),
"direct peer has no connection to close"
);
for conn_id in conn_ids {
let _ = inst
.get_core_instance()
.close_peer_conn(peer_id, &conn_id)
.await;
}
wait_for_condition(
|| async {
!inst
.get_core_instance()
.connected_peers()
.await
.contains(&peer_id)
},
Duration::from_secs(10),
)
.await;
}
/// Prepare network namespaces for credential tests
/// Topology:
/// br_a (10.1.1.0/24): ns_adm (10.1.1.1), ns_c1 (10.1.1.2), ns_c2 (10.1.1.3), ns_c3 (10.1.1.4), ns_c4 (10.1.1.5)
/// br_b (10.1.2.0/24): ns_adm2 (10.1.2.1) - for multi-admin tests
/// Note: Using short names (max 15 chars for veth interfaces)
fn prepare_credential_network() -> TestNetwork {
let mut network = TestNetwork::new();
// Create bridge br_a for admin and credentials
network.create_bridge("br_a");
// Create namespaces and add to bridge
network.create_namespace("ns_adm", "10.1.1.1/24", "fd11::1/64");
network.add_to_bridge("br_a", "ns_adm");
network.create_namespace("ns_c1", "10.1.1.2/24", "fd11::2/64");
network.add_to_bridge("br_a", "ns_c1");
network.create_namespace("ns_c2", "10.1.1.3/24", "fd11::3/64");
network.add_to_bridge("br_a", "ns_c2");
// Create ns_c3 for relay tests (needs 4 nodes)
network.create_namespace("ns_c3", "10.1.1.4/24", "fd11::4/64");
network.add_to_bridge("br_a", "ns_c3");
// Create ns_c4 for multi-admin credential tests (needs 5 nodes)
network.create_namespace("ns_c4", "10.1.1.5/24", "fd11::5/64");
network.add_to_bridge("br_a", "ns_c4");
// Create bridge br_b for second admin (multi-admin tests)
network.create_bridge("br_b");
network.create_namespace("ns_adm2", "10.1.2.1/24", "fd12::1/64");
network.add_to_bridge("br_b", "ns_adm2");
network
}
fn credential_private_key_from_secret(credential_secret: &str) -> x25519_dalek::StaticSecret {
use base64::Engine as _;
let privkey_bytes: [u8; 32] = base64::prelude::BASE64_STANDARD
.decode(credential_secret)
.unwrap()
.try_into()
.unwrap();
x25519_dalek::StaticSecret::from(privkey_bytes)
}
fn build_credential_config(
network_name: String,
private_key: &x25519_dalek::StaticSecret,
inst_name: &str,
ns: Option<&str>,
ipv4: &str,
ipv6: &str,
) -> TomlConfigLoader {
let config = TomlConfigLoader::default();
config.set_inst_name(inst_name.to_owned());
config.set_netns(ns.map(|s| s.to_owned()));
config.set_ipv4(Some(ipv4.parse().unwrap()));
config.set_ipv6(Some(ipv6.parse().unwrap()));
config.set_listeners(vec![]);
config.set_network_identity(NetworkIdentity::new_credential(network_name));
config.set_secure_mode(Some(generate_secure_mode_config_with_key(private_key)));
config
}
/// Helper: Create credential node config with generated credential
async fn create_credential_config(
admin_inst: &Instance,
inst_name: &str,
ns: Option<&str>,
ipv4: &str,
ipv6: &str,
) -> TomlConfigLoader {
let (_cred_id, cred_secret) =
generate_credential(admin_inst, vec![], false, vec![], Duration::from_secs(3600)).await;
build_credential_config(
admin_inst
.get_global_ctx()
.get_network_identity()
.network_name
.clone(),
&credential_private_key_from_secret(&cred_secret),
inst_name,
ns,
ipv4,
ipv6,
)
}
fn create_credential_config_from_secret(
network_name: String,
credential_secret: &str,
inst_name: &str,
ns: Option<&str>,
ipv4: &str,
ipv6: &str,
) -> TomlConfigLoader {
build_credential_config(
network_name,
&credential_private_key_from_secret(credential_secret),
inst_name,
ns,
ipv4,
ipv6,
)
}
/// Helper: Create credential node config with a random, unknown key
fn create_unknown_credential_config(
network_name: String,
inst_name: &str,
ns: Option<&str>,
ipv4: &str,
ipv6: &str,
) -> TomlConfigLoader {
let random_private = x25519_dalek::StaticSecret::random_from_rng(rand::rngs::OsRng);
build_credential_config(network_name, &random_private, inst_name, ns, ipv4, ipv6)
}
/// Helper: Create admin node config
fn create_admin_config(
inst_name: &str,
ns: Option<&str>,
ipv4: &str,
ipv6: &str,
) -> TomlConfigLoader {
let config = TomlConfigLoader::default();
config.set_inst_name(inst_name.to_owned());
config.set_netns(ns.map(|s| s.to_owned()));
config.set_ipv4(Some(ipv4.parse().unwrap()));
config.set_ipv6(Some(ipv6.parse().unwrap()));
config.set_listeners(vec![
"tcp://0.0.0.0:11010".parse().unwrap(),
"udp://0.0.0.0:11010".parse().unwrap(),
]);
config.set_network_identity(NetworkIdentity::new(
"test_network".to_string(),
"test_secret".to_string(),
));
config.set_secure_mode(Some(generate_secure_mode_config()));
config
}
fn create_shared_config(
inst_name: &str,
ns: Option<&str>,
ipv4: &str,
ipv6: &str,
) -> TomlConfigLoader {
let config = TomlConfigLoader::default();
config.set_inst_name(inst_name.to_owned());
config.set_netns(ns.map(|s| s.to_owned()));
config.set_ipv4(Some(ipv4.parse().unwrap()));
config.set_ipv6(Some(ipv6.parse().unwrap()));
config.set_listeners(vec![
"tcp://0.0.0.0:11010".parse().unwrap(),
"udp://0.0.0.0:11010".parse().unwrap(),
]);
config.set_network_identity(NetworkIdentity::new(
"shared_network".to_string(),
"".to_string(),
));
config.set_secure_mode(Some(generate_secure_mode_config()));
config
}
fn create_public_server_config(network: &TestNetwork) -> TomlConfigLoader {
let config = TomlConfigLoader::default();
config.set_inst_name(PUBLIC_SERVER_NETWORK_NAME.to_string());
config.set_hostname(Some("public-server".to_string()));
config.set_netns(Some(network.namespace("ns_adm").to_string()));
config.set_listeners(vec!["udp://0.0.0.0:11010".parse().unwrap()]);
config.set_network_identity(NetworkIdentity::new(
PUBLIC_SERVER_NETWORK_NAME.to_string(),
PUBLIC_SERVER_SHARED_SECRET.to_string(),
));
config.set_secure_mode(Some(generate_secure_mode_config()));
let mut flags = config.get_flags();
flags.no_tun = true;
flags.private_mode = true;
flags.relay_all_peer_rpc = true;
flags.relay_network_whitelist = "".to_string();
config.set_flags(flags);
config
}
fn create_need_p2p_admin_config(network: &TestNetwork, listener_scheme: &str) -> TomlConfigLoader {
let config = TomlConfigLoader::default();
config.set_inst_name(NEED_P2P_ADMIN_NETWORK_NAME.to_string());
config.set_hostname(Some("need-p2p-admin".to_string()));
config.set_netns(Some(network.namespace("ns_c3").to_string()));
config.set_listeners(vec![
format!("{listener_scheme}://0.0.0.0:0").parse().unwrap(),
]);
config.set_network_identity(NetworkIdentity::new(
NEED_P2P_ADMIN_NETWORK_NAME.to_string(),
PUBLIC_SERVER_SHARED_SECRET.to_string(),
));
config.set_secure_mode(Some(generate_secure_mode_config()));
let mut flags = config.get_flags();
flags.no_tun = true;
flags.relay_all_peer_rpc = true;
flags.need_p2p = true;
flags.disable_udp_hole_punching = true;
flags.disable_tcp_hole_punching = true;
flags.disable_sym_hole_punching = true;
config.set_flags(flags);
config
}
#[allow(clippy::too_many_arguments)]
fn create_public_server_credential_config(
credential_secret: &str,
inst_name: &str,
hostname: &str,
ns: &str,
ipv4: &str,
ipv6: &str,
tcp_listener_port: u16,
udp_listener_port: u16,
proxy_cidrs: &[&str],
) -> TomlConfigLoader {
let config = create_credential_config_from_secret(
NEED_P2P_ADMIN_NETWORK_NAME.to_string(),
credential_secret,
inst_name,
Some(ns),
ipv4,
ipv6,
);
config.set_hostname(Some(hostname.to_string()));
config.set_listeners(vec![
format!("tcp://0.0.0.0:{tcp_listener_port}")
.parse()
.unwrap(),
format!("udp://0.0.0.0:{udp_listener_port}")
.parse()
.unwrap(),
]);
for cidr in proxy_cidrs {
config
.add_proxy_cidr((*cidr).parse().unwrap(), None)
.unwrap();
}
let mut flags = config.get_flags();
flags.disable_p2p = true;
config.set_flags(flags);
config
}
async fn wait_direct_peer(inst: &Instance, peer_id: u32, timeout: Duration, label: &str) {
wait_for_condition(
|| async {
let peers = inst.get_core_instance().connected_peers().await;
let connected = peers.contains(&peer_id);
println!("{label}: direct peers={:?}, target={}", peers, peer_id);
connected
},
timeout,
)
.await;
}
async fn wait_running_listener(inst: &Instance, scheme: &str, timeout: Duration, label: &str) {
wait_for_condition(
|| async {
let listeners = inst.get_core_instance().running_listeners();
let matched = listeners.iter().any(|listener| {
listener.scheme() == scheme && listener.port().is_some_and(|p| p != 0)
});
println!("{label}: running listeners={:?}", listeners);
matched
},
timeout,
)
.await;
}
async fn wait_route_cost(inst: &Instance, peer_id: u32, cost: i32, timeout: Duration, label: &str) {
wait_for_condition(
|| async {
let routes = inst.get_core_instance().route_snapshots().await;
let matched = routes
.iter()
.any(|route| route.peer_id == peer_id && route.cost == cost);
println!(
"{label}: routes={:?}, target={}, cost={}",
routes
.iter()
.map(|route| (route.peer_id, route.cost))
.collect::<Vec<_>>(),
peer_id,
cost
);
matched
},
timeout,
)
.await;
}
async fn wait_foreign_network_count(inst: &Instance, expected: usize, timeout: Duration) {
wait_for_condition(
|| async {
let foreign_networks = inst
.get_core_instance()
.foreign_network_snapshots(false)
.await;
println!("foreign networks: {:?}", foreign_networks);
foreign_networks.len() == expected
},
timeout,
)
.await;
}
/// Regression coverage for a public-server-mediated credential topology:
/// Public server <- admin peer (need_p2p) <- two credential peers.
///
/// Credential peers set `disable_p2p=true`, while the admin peer advertises `need_p2p=true`.
/// The credential peers should still proactively build direct peers with the admin peer through
/// peer RPC forwarded by the public server, even when the admin listener binds an ephemeral port.
#[rstest]
#[case("quic")]
#[case("wss")]
#[case("tcp")]
#[case("udp")]
#[tokio::test]
#[serial_test::serial]
async fn credential_peers_p2p_to_need_p2p_admin_through_public_server(
#[case] admin_listener_scheme: &str,
) {
let network = prepare_credential_network();
let process_runtime = CoreProcessRuntime::new();
let mut public_server_inst = Instance::new_with_process_runtime(
create_public_server_config(&network),
process_runtime.clone(),
);
public_server_inst.run().await.unwrap();
let mut admin_inst = Instance::new_with_process_runtime(
create_need_p2p_admin_config(&network, admin_listener_scheme),
process_runtime.clone(),
);
admin_inst.run().await.unwrap();
wait_running_listener(
&admin_inst,
admin_listener_scheme,
Duration::from_secs(10),
"admin ephemeral listener",
)
.await;
admin_inst.add_connector_url("udp://10.1.1.1:11010".parse().unwrap());
wait_foreign_network_count(&public_server_inst, 1, Duration::from_secs(10)).await;
let (_credential_a_id, credential_a_secret) = generate_credential_with_options(
&admin_inst,
vec![],
false,
vec!["10.1.0.0/24".to_string()],
Duration::from_secs(3600),
Some("credential-peer-a".to_string()),
false,
)
.await;
let (_credential_b_id, credential_b_secret) = generate_credential_with_options(
&admin_inst,
vec![],
false,
vec![],
Duration::from_secs(3600),
Some("credential-peer-b".to_string()),
false,
)
.await;
admin_inst
.get_global_ctx()
.issue_event(GlobalCtxEvent::CredentialChanged);
wait_foreign_network_count(&public_server_inst, 1, Duration::from_secs(10)).await;
let mut credential_a_inst = Instance::new_with_process_runtime(
create_public_server_credential_config(
&credential_a_secret,
"credential-peer-a",
"credential-a",
network.namespace("ns_c1"),
"10.154.0.1",
"fd00::1/64",
11030,
11031,
&["10.1.0.0/24"],
),
process_runtime.clone(),
);
let mut credential_b_inst = Instance::new_with_process_runtime(
create_public_server_credential_config(
&credential_b_secret,
"credential-peer-b",
"credential-b",
network.namespace("ns_c2"),
"10.154.0.2",
"fd00::2/64",
11040,
11041,
&[],
),
process_runtime.clone(),
);
credential_a_inst.run().await.unwrap();
credential_b_inst.run().await.unwrap();
credential_a_inst.add_connector_url("udp://10.1.1.1:11010".parse().unwrap());
credential_b_inst.add_connector_url("udp://10.1.1.1:11010".parse().unwrap());
let admin_peer_id = admin_inst.peer_id();
let credential_a_peer_id = credential_a_inst.peer_id();
let credential_b_peer_id = credential_b_inst.peer_id();
println!(
"admin={}, credential_a={}, credential_b={}, admin_listener_scheme={}",
admin_peer_id, credential_a_peer_id, credential_b_peer_id, admin_listener_scheme
);
wait_direct_peer(
&credential_a_inst,
admin_peer_id,
Duration::from_secs(30),
"credential_a -> admin",
)
.await;
wait_direct_peer(
&credential_b_inst,
admin_peer_id,
Duration::from_secs(30),
"credential_b -> admin",
)
.await;
wait_direct_peer(
&admin_inst,
credential_a_peer_id,
Duration::from_secs(10),
"admin -> credential_a",
)
.await;
wait_direct_peer(
&admin_inst,
credential_b_peer_id,
Duration::from_secs(10),
"admin -> credential_b",
)
.await;
wait_route_cost(
&credential_a_inst,
admin_peer_id,
1,
Duration::from_secs(10),
"credential_a route to admin",
)
.await;
wait_route_cost(
&credential_b_inst,
admin_peer_id,
1,
Duration::from_secs(10),
"credential_b route to admin",
)
.await;
drop_insts(vec![
public_server_inst,
admin_inst,
credential_a_inst,
credential_b_inst,
])
.await;
}
#[cfg(feature = "wireguard")]
#[tokio::test]
#[serial_test::serial]
async fn credential_peer_reconnects_to_admin_with_portal_client_online() {
let network = prepare_credential_network();
let process_runtime = CoreProcessRuntime::new();
let public_server_config = create_public_server_config(&network);
let mut public_server_flags = public_server_config.get_flags();
public_server_flags.disable_relay_data = true;
public_server_config.set_flags(public_server_flags);
let mut public_server_inst =
Instance::new_with_process_runtime(public_server_config, process_runtime.clone());
public_server_inst.run().await.unwrap();
let admin_config = create_need_p2p_admin_config(&network, "udp");
admin_config.set_vpn_portal_config(VpnPortalConfig {
enabled: None,
wireguard_listen: "0.0.0.0:22121".parse().unwrap(),
wireguard_private_key: Some(BASE64_STANDARD.encode([42u8; 32])),
clients: vec![VpnPortalClientConfig {
name: "portal-client".to_owned(),
virtual_ip: "10.154.0.10/24".parse().unwrap(),
groups: Vec::new(),
}],
});
let mut admin_inst = Instance::new_with_process_runtime(admin_config, process_runtime.clone());
admin_inst.run().await.unwrap();
admin_inst.add_connector_url("udp://10.1.1.1:11010".parse().unwrap());
wait_foreign_network_count(&public_server_inst, 1, Duration::from_secs(10)).await;
let (_credential_id, credential_secret) = generate_credential_with_options(
&admin_inst,
Vec::new(),
false,
Vec::new(),
Duration::from_secs(3600),
Some("portal-p2p-credential".to_owned()),
false,
)
.await;
admin_inst
.get_global_ctx()
.issue_event(GlobalCtxEvent::CredentialChanged);
let credential_config = create_public_server_credential_config(
&credential_secret,
"portal-credential-peer",
"portal-credential-peer",
network.namespace("ns_c1"),
"10.154.0.1",
"fd00::1/64",
11030,
11031,
&[],
);
let mut credential_inst =
Instance::new_with_process_runtime(credential_config, process_runtime);
credential_inst.run().await.unwrap();
credential_inst.add_connector_url("udp://10.1.1.1:11010".parse().unwrap());
let admin_peer_id = admin_inst.peer_id();
let credential_peer_id = credential_inst.peer_id();
wait_direct_peer(
&credential_inst,
admin_peer_id,
Duration::from_secs(30),
"credential -> admin before portal client",
)
.await;
wait_direct_peer(
&admin_inst,
credential_peer_id,
Duration::from_secs(10),
"admin -> credential before portal client",
)
.await;
wait_route_cost(
&credential_inst,
admin_peer_id,
1,
Duration::from_secs(10),
"credential route to admin before portal client",
)
.await;
let portal_config = admin_inst
.get_global_ctx()
.config
.get_vpn_portal_config()
.unwrap();
let (server_public, client_private) =
test_wireguard_keys(&portal_config, "portal-client").unwrap();
{
let net_ns = crate::common::netns::NetNS::new(Some(network.namespace("ns_c4").to_owned()));
let _guard = net_ns.guard();
run_wireguard_client(
&wireguard_ifname("wg0"),
"10.1.1.4:22121".parse().unwrap(),
Key::try_from(server_public.as_slice()).unwrap(),
Key::try_from(client_private.as_slice()).unwrap(),
vec!["10.154.0.0/24".to_owned()],
"10.154.0.10".to_owned(),
)
.unwrap();
}
let public_server_forwarded_before =
forwarded_data_packets_for_network(&public_server_inst, NEED_P2P_ADMIN_NETWORK_NAME);
wait_for_condition(
|| async {
ping_test(network.namespace("ns_c4"), "10.154.0.1", None).await;
admin_inst
.get_core_instance()
.vpn_portal_info()
.await
.clients
.iter()
.any(|client| client.state == PortalClientState::Online)
},
Duration::from_secs(10),
)
.await;
let old_conn_ids = credential_inst
.get_core_instance()
.peer_snapshots()
.await
.into_iter()
.find(|peer| peer.peer_id == admin_peer_id)
.map(|peer| peer.directly_connected_conns)
.unwrap_or_default();
assert!(
!old_conn_ids.is_empty(),
"credential peer must have a direct admin connection to replace"
);
for conn_id in &old_conn_ids {
credential_inst
.get_core_instance()
.close_peer_conn(admin_peer_id, conn_id)
.await
.unwrap();
}
wait_for_condition(
|| async {
let has_new_connection = credential_inst
.get_core_instance()
.peer_snapshots()
.await
.into_iter()
.find(|peer| peer.peer_id == admin_peer_id)
.is_some_and(|peer| {
peer.directly_connected_conns
.iter()
.any(|conn_id| !old_conn_ids.contains(conn_id))
});
let has_direct_route = credential_inst
.get_core_instance()
.route_snapshots()
.await
.iter()
.any(|route| {
route.peer_id == admin_peer_id
&& route.next_hop_peer_id == admin_peer_id
&& route.cost == 1
});
has_new_connection && has_direct_route
},
Duration::from_secs(30),
)
.await;
wait_ping_reachability(
network.namespace("ns_c1"),
"10.154.0.10",
true,
Duration::from_secs(10),
)
.await;
for _ in 0..3 {
assert!(ping_test(network.namespace("ns_c1"), "10.154.0.10", None).await);
}
assert_eq!(
forwarded_data_packets_for_network(&public_server_inst, NEED_P2P_ADMIN_NETWORK_NAME),
public_server_forwarded_before,
"public server forwarded data despite disable_relay_data"
);
drop_insts(vec![public_server_inst, admin_inst, credential_inst]).await;
}
async fn create_generated_credential_config(
admin_inst: &Instance,
inst_name: &str,
ns: Option<&str>,
ipv4: &str,
ipv6: &str,
) -> (TomlConfigLoader, String) {
let (cred_id, cred_secret) =
generate_credential(admin_inst, vec![], false, vec![], Duration::from_secs(3600)).await;
let config = build_credential_config(
admin_inst
.get_global_ctx()
.get_network_identity()
.network_name
.clone(),
&credential_private_key_from_secret(&cred_secret),
inst_name,
ns,
ipv4,
ipv6,
);
(config, cred_id)
}
async fn wait_ping_reachability(src_ns: &str, dst_ip: &str, reachable: bool, timeout: Duration) {
wait_for_condition(
|| async {
let ping_result = ping_test(src_ns, dst_ip, None).await;
if reachable { ping_result } else { !ping_result }
},
timeout,
)
.await;
}
async fn wait_route_presence_on_admins(
admin_a_inst: &Instance,
admin_c_inst: &Instance,
peer_id: u32,
should_exist: bool,
timeout: Duration,
) {
wait_for_condition(
|| async {
let admin_a_routes = admin_a_inst.get_core_instance().route_snapshots().await;
let admin_c_routes = admin_c_inst.get_core_instance().route_snapshots().await;
let admin_a_has = admin_a_routes.iter().any(|r| r.peer_id == peer_id);
let admin_c_has = admin_c_routes.iter().any(|r| r.peer_id == peer_id);
if should_exist {
admin_a_has || admin_c_has
} else {
!admin_a_has && !admin_c_has
}
},
timeout,
)
.await;
}
async fn assert_shared_visibility_stable(
network: &TestNetwork,
admin_a_inst: &Instance,
admin_c_inst: &Instance,
peer_id: u32,
peer_ip: &str,
should_exist: bool,
label: &str,
) {
for _ in 0..5 {
let admin_a_routes = admin_a_inst.get_core_instance().route_snapshots().await;
let admin_c_routes = admin_c_inst.get_core_instance().route_snapshots().await;
let admin_a_has = admin_a_routes.iter().any(|r| r.peer_id == peer_id);
let admin_c_has = admin_c_routes.iter().any(|r| r.peer_id == peer_id);
if should_exist {
assert!(
admin_a_has || admin_c_has,
"{} should exist via shared path but routes are admin_a={:?} admin_c={:?}",
label,
admin_a_routes.iter().map(|r| r.peer_id).collect::<Vec<_>>(),
admin_c_routes.iter().map(|r| r.peer_id).collect::<Vec<_>>()
);
} else {
assert!(
!admin_a_has && !admin_c_has,
"{} should be absent via shared path but routes are admin_a={:?} admin_c={:?}",
label,
admin_a_routes.iter().map(|r| r.peer_id).collect::<Vec<_>>(),
admin_c_routes.iter().map(|r| r.peer_id).collect::<Vec<_>>()
);
}
let (ping_from_admin_a, ping_from_admin_c) = tokio::join!(
ping_test(network.namespace("ns_adm"), peer_ip, None),
ping_test(network.namespace("ns_c3"), peer_ip, None),
);
if should_exist {
assert!(ping_from_admin_a, "admin_a should reach {}", label);
assert!(ping_from_admin_c, "admin_c should reach {}", label);
} else {
assert!(!ping_from_admin_a, "admin_a should not reach {}", label);
assert!(!ping_from_admin_c, "admin_c should not reach {}", label);
}
tokio::time::sleep(Duration::from_secs(1)).await;
}
}
async fn wait_stable_single_visible_peer_on_admins(
admin_a_inst: &Instance,
admin_c_inst: &Instance,
peer_a_id: u32,
peer_b_id: u32,
timeout: Duration,
) -> u32 {
let start = std::time::Instant::now();
let mut stable_winner = None;
let mut stable_samples = 0;
loop {
let admin_a_routes = admin_a_inst.get_core_instance().route_snapshots().await;
let admin_c_routes = admin_c_inst.get_core_instance().route_snapshots().await;
let admin_a_has_a = admin_a_routes.iter().any(|r| r.peer_id == peer_a_id);
let admin_a_has_b = admin_a_routes.iter().any(|r| r.peer_id == peer_b_id);
let admin_c_has_a = admin_c_routes.iter().any(|r| r.peer_id == peer_a_id);
let admin_c_has_b = admin_c_routes.iter().any(|r| r.peer_id == peer_b_id);
let current_winner = if admin_a_has_a && admin_c_has_a && !admin_a_has_b && !admin_c_has_b {
Some(peer_a_id)
} else if admin_a_has_b && admin_c_has_b && !admin_a_has_a && !admin_c_has_a {
Some(peer_b_id)
} else {
None
};
println!(
"single-visible routes: a={:?} c={:?} winner={:?}",
admin_a_routes.iter().map(|r| r.peer_id).collect::<Vec<_>>(),
admin_c_routes.iter().map(|r| r.peer_id).collect::<Vec<_>>(),
current_winner
);
match current_winner {
Some(winner) if stable_winner == Some(winner) => stable_samples += 1,
Some(winner) => {
stable_winner = Some(winner);
stable_samples = 1;
}
None => {
stable_winner = None;
stable_samples = 0;
}
}
if stable_samples >= 3 {
return stable_winner.unwrap();
}
assert!(
start.elapsed() < timeout,
"timed out waiting for a stable single visible peer on both admins after {:?}: a_has_a={} a_has_b={} c_has_a={} c_has_b={} a={:?} c={:?}",
start.elapsed(),
admin_a_has_a,
admin_a_has_b,
admin_c_has_a,
admin_c_has_b,
admin_a_routes.iter().map(|r| r.peer_id).collect::<Vec<_>>(),
admin_c_routes.iter().map(|r| r.peer_id).collect::<Vec<_>>()
);
tokio::time::sleep(Duration::from_secs(1)).await;
}
}
async fn wait_stable_failover_visibility_on_admins(
admin_a_inst: &Instance,
admin_c_inst: &Instance,
present_peer_id: u32,
absent_peer_id: u32,
timeout: Duration,
) {
let start = std::time::Instant::now();
let mut stable_samples = 0;
loop {
let admin_a_routes = admin_a_inst.get_core_instance().route_snapshots().await;
let admin_c_routes = admin_c_inst.get_core_instance().route_snapshots().await;
let admin_a_has_present = admin_a_routes.iter().any(|r| r.peer_id == present_peer_id);
let admin_c_has_present = admin_c_routes.iter().any(|r| r.peer_id == present_peer_id);
let admin_a_has_absent = admin_a_routes.iter().any(|r| r.peer_id == absent_peer_id);
let admin_c_has_absent = admin_c_routes.iter().any(|r| r.peer_id == absent_peer_id);
let failover_stable = admin_a_has_present
&& admin_c_has_present
&& !admin_a_has_absent
&& !admin_c_has_absent;
println!(
"failover visibility: present={} a_has_present={} c_has_present={} absent={} a_has_absent={} c_has_absent={} stable={} samples={}",
present_peer_id,
admin_a_has_present,
admin_c_has_present,
absent_peer_id,
admin_a_has_absent,
admin_c_has_absent,
failover_stable,
stable_samples
);
if failover_stable {
stable_samples += 1;
} else {
stable_samples = 0;
}
if stable_samples >= 3 {
return;
}
assert!(
start.elapsed() < timeout,
"timed out waiting for stable failover visibility on both admins after {:?}: present={} a_has_present={} c_has_present={} absent={} a_has_absent={} c_has_absent={} a={:?} c={:?}",
start.elapsed(),
present_peer_id,
admin_a_has_present,
admin_c_has_present,
absent_peer_id,
admin_a_has_absent,
admin_c_has_absent,
admin_a_routes.iter().map(|r| r.peer_id).collect::<Vec<_>>(),
admin_c_routes.iter().map(|r| r.peer_id).collect::<Vec<_>>()
);
tokio::time::sleep(Duration::from_secs(1)).await;
}
}
/// Test 1: Basic credential node connectivity with and without an admin key pin
/// Topology: Admin ← Credential
/// Verifies that a credential node can connect to an admin node and appears in routes
#[rstest]
#[case(false)]
#[case(true)]
#[tokio::test]
#[serial_test::serial]
async fn credential_basic_connectivity(#[case] pin_admin: bool) {
let network = prepare_credential_network();
let process_runtime = CoreProcessRuntime::new();
// Create admin node
let admin_config = create_admin_config(
"admin",
Some(network.namespace("ns_adm")),
"10.144.144.1",
"fd00::1/64",
);
let admin_public_key = admin_config
.get_secure_mode()
.and_then(|config| config.local_public_key)
.unwrap();
let mut admin_inst = Instance::new_with_process_runtime(admin_config, process_runtime.clone());
admin_inst.run().await.unwrap();
// Create credential node
let cred_config = create_credential_config(
&admin_inst,
"cred",
Some(network.namespace("ns_c1")),
"10.144.144.2",
"fd00::2/64",
)
.await;
if pin_admin {
cred_config.set_peers(vec![PeerConfig {
uri: "tcp://10.1.1.1:11010".parse().unwrap(),
peer_public_key: Some(admin_public_key),
}]);
}
let mut cred_inst = Instance::new_with_process_runtime(cred_config, process_runtime.clone());
cred_inst.run().await.unwrap();
// Credential connects to admin
if !pin_admin {
cred_inst.add_connector_url("tcp://10.1.1.1:11010".parse().unwrap());
}
let cred_peer_id = cred_inst.peer_id();
let admin_peer_id = admin_inst.peer_id();
println!(
"Admin peer_id: {}, Credential peer_id: {}",
admin_peer_id, cred_peer_id
);
// Wait a bit for connection attempt
tokio::time::sleep(Duration::from_secs(2)).await;
// Check peers and connections
let admin_peers = admin_inst.get_core_instance().connected_peers().await;
let cred_peers = cred_inst.get_core_instance().connected_peers().await;
println!("Admin peers: {:?}", admin_peers);
println!("Credential peers: {:?}", cred_peers);
// Wait for credential to appear in admin's route table
wait_for_condition(
|| async {
let routes = admin_inst.get_core_instance().route_snapshots().await;
let cred_routes = cred_inst.get_core_instance().route_snapshots().await;
let admin_peers = admin_inst.get_core_instance().connected_peers().await;
let cred_peers = cred_inst.get_core_instance().connected_peers().await;
println!(
"Admin peers: {:?}, routes: {:?}",
admin_peers,
routes
.iter()
.map(|r| (r.peer_id, r.ipv4_addr))
.collect::<Vec<_>>()
);
println!(
"Credential peers: {:?}, routes: {:?}",
cred_peers,
cred_routes
.iter()
.map(|r| (r.peer_id, r.ipv4_addr))
.collect::<Vec<_>>()
);
routes.iter().any(|r| r.peer_id == cred_peer_id)
},
Duration::from_secs(10),
)
.await;
// Verify connectivity
wait_for_condition(
|| async { ping_test(network.namespace("ns_adm"), "10.144.144.2", None).await },
Duration::from_secs(10),
)
.await;
wait_for_condition(
|| async { ping_test(network.namespace("ns_c1"), "10.144.144.1", None).await },
Duration::from_secs(10),
)
.await;
drop_insts(vec![admin_inst, cred_inst]).await;
}
/// A credential node must reject an admin whose Noise key does not match its pin.
#[tokio::test]
#[serial_test::serial]
async fn credential_rejects_incorrect_admin_pin() {
let network = prepare_credential_network();
let process_runtime = CoreProcessRuntime::new();
let admin_config = create_admin_config(
"admin",
Some(network.namespace("ns_adm")),
"10.144.144.1",
"fd00::1/64",
);
let admin_public_key = admin_config
.get_secure_mode()
.and_then(|config| config.local_public_key)
.unwrap();
let mut admin_inst = Instance::new_with_process_runtime(admin_config, process_runtime.clone());
admin_inst.run().await.unwrap();
let cred_config = create_credential_config(
&admin_inst,
"cred",
Some(network.namespace("ns_c1")),
"10.144.144.2",
"fd00::2/64",
)
.await;
let incorrect_admin_public_key = generate_secure_mode_config().local_public_key.unwrap();
assert_ne!(incorrect_admin_public_key, admin_public_key);
cred_config.set_peers(vec![PeerConfig {
uri: "tcp://10.1.1.1:11010".parse().unwrap(),
peer_public_key: Some(incorrect_admin_public_key),
}]);
let mut cred_inst = Instance::new_with_process_runtime(cred_config, process_runtime.clone());
cred_inst.run().await.unwrap();
let admin_peer_id = admin_inst.peer_id();
let cred_peer_id = cred_inst.peer_id();
for _ in 0..5 {
let admin_peers = admin_inst.get_core_instance().connected_peers().await;
let cred_peers = cred_inst.get_core_instance().connected_peers().await;
let admin_routes = admin_inst.get_core_instance().route_snapshots().await;
let cred_routes = cred_inst.get_core_instance().route_snapshots().await;
assert!(!admin_peers.contains(&cred_peer_id));
assert!(!cred_peers.contains(&admin_peer_id));
assert!(
!admin_routes
.iter()
.any(|route| route.peer_id == cred_peer_id)
);
assert!(
!cred_routes
.iter()
.any(|route| route.peer_id == admin_peer_id)
);
tokio::time::sleep(Duration::from_secs(1)).await;
}
drop_insts(vec![admin_inst, cred_inst]).await;
}
/// Test 5-6: Prefer Peer Relay with one admin and an allow_relay credential
/// Topology: Admin ← Credential_A, Admin ← Credential_B, Admin ← Credential_C(listener, allow_relay)
/// Verifies routing behavior based on allow_relay flag:
/// - allow_relay=true: A→B route goes through C (cost 2 via C)
/// - allow_relay=false: A→B route goes through Admin (cost 2 via Admin)
#[rstest]
#[case(true)]
#[case(false)]
#[tokio::test]
#[serial_test::serial]
async fn prefer_peer_relay_single_admin(#[case] allow_relay: bool) {
let network = prepare_credential_network();
let process_runtime = CoreProcessRuntime::new();
// Create admin node
let admin_config = create_admin_config(
"admin",
Some(network.namespace("ns_adm")),
"10.144.144.1",
"fd00::1/64",
);
let mut admin_inst = Instance::new_with_process_runtime(admin_config, process_runtime.clone());
admin_inst.run().await.unwrap();
let admin_peer_id = admin_inst.peer_id();
// Generate credentials for A, B, C
// C has configurable allow_relay
let (_cred_a_id, cred_a_secret) = generate_credential(
&admin_inst,
vec![],
false,
vec![],
Duration::from_secs(3600),
)
.await;
let (_cred_b_id, cred_b_secret) = generate_credential(
&admin_inst,
vec![],
false,
vec![],
Duration::from_secs(3600),
)
.await;
let (_cred_c_id, cred_c_secret) = generate_credential(
&admin_inst,
vec![],
allow_relay,
vec![],
Duration::from_secs(3600),
)
.await;
// Create credential A on ns_c1
let cred_a_config = {
use base64::Engine as _;
let privkey_bytes: [u8; 32] = base64::prelude::BASE64_STANDARD
.decode(&cred_a_secret)
.unwrap()
.try_into()
.unwrap();
let private = x25519_dalek::StaticSecret::from(privkey_bytes);
let config = TomlConfigLoader::default();
config.set_inst_name("cred_a".to_string());
config.set_netns(Some(network.namespace("ns_c1").to_string()));
config.set_ipv4(Some("10.144.144.2".parse().unwrap()));
config.set_ipv6(Some("fd00::2/64".parse().unwrap()));
config.set_network_identity(NetworkIdentity::new_credential(
admin_inst
.get_global_ctx()
.get_network_identity()
.network_name
.clone(),
));
config.set_secure_mode(Some(generate_secure_mode_config_with_key(&private)));
disable_p2p(&config);
config
};
let mut cred_a_inst =
Instance::new_with_process_runtime(cred_a_config, process_runtime.clone());
cred_a_inst.run().await.unwrap();
// Create credential B on ns_c2
let cred_b_config = {
use base64::Engine as _;
let privkey_bytes: [u8; 32] = base64::prelude::BASE64_STANDARD
.decode(&cred_b_secret)
.unwrap()
.try_into()
.unwrap();
let private = x25519_dalek::StaticSecret::from(privkey_bytes);
let config = TomlConfigLoader::default();
config.set_inst_name("cred_b".to_string());
config.set_netns(Some(network.namespace("ns_c2").to_string()));
config.set_ipv4(Some("10.144.144.3".parse().unwrap()));
config.set_ipv6(Some("fd00::3/64".parse().unwrap()));
config.set_network_identity(NetworkIdentity::new_credential(
admin_inst
.get_global_ctx()
.get_network_identity()
.network_name
.clone(),
));
config.set_secure_mode(Some(generate_secure_mode_config_with_key(&private)));
disable_p2p(&config);
config
};
let mut cred_b_inst =
Instance::new_with_process_runtime(cred_b_config, process_runtime.clone());
cred_b_inst.run().await.unwrap();
// Create credential C on ns_c3 WITH listener (so A and B can connect to it)
let cred_c_config = {
use base64::Engine as _;
let privkey_bytes: [u8; 32] = base64::prelude::BASE64_STANDARD
.decode(&cred_c_secret)
.unwrap()
.try_into()
.unwrap();
let private = x25519_dalek::StaticSecret::from(privkey_bytes);
let config = TomlConfigLoader::default();
config.set_inst_name("cred_c".to_string());
config.set_netns(Some(network.namespace("ns_c3").to_string()));
config.set_ipv4(Some("10.144.144.4".parse().unwrap()));
config.set_ipv6(Some("fd00::4/64".parse().unwrap()));
// C has listener so A and B can connect to it
config.set_listeners(vec!["tcp://0.0.0.0:11020".parse().unwrap()]);
config.set_network_identity(NetworkIdentity::new_credential(
admin_inst
.get_global_ctx()
.get_network_identity()
.network_name
.clone(),
));
config.set_secure_mode(Some(generate_secure_mode_config_with_key(&private)));
disable_p2p(&config);
config
};
let mut cred_c_inst =
Instance::new_with_process_runtime(cred_c_config, process_runtime.clone());
cred_c_inst.run().await.unwrap();
let cred_a_peer_id = cred_a_inst.peer_id();
let cred_b_peer_id = cred_b_inst.peer_id();
let cred_c_peer_id = cred_c_inst.peer_id();
// All credentials connect to admin
cred_a_inst.add_connector_url("tcp://10.1.1.1:11010".parse().unwrap());
cred_b_inst.add_connector_url("tcp://10.1.1.1:11010".parse().unwrap());
cred_c_inst.add_connector_url("tcp://10.1.1.1:11010".parse().unwrap());
// B connects to C first. A remains Admin-only so changing the Admin's
// advertised edge produces an observable 2 -> 3 -> 2 transition.
cred_b_inst.add_connector_url("tcp://10.1.1.4:11020".parse().unwrap());
// print all peer ids
println!("Admin peer id: {:?}", admin_peer_id);
println!("Cred A peer id: {:?}", cred_a_peer_id);
println!("Cred B peer id: {:?}", cred_b_peer_id);
println!("Cred C peer id: {:?}", cred_c_peer_id);
// Wait for all nodes to appear in admin's route table
wait_for_condition(
|| async {
let routes = admin_inst.get_core_instance().route_snapshots().await;
let has_a = routes.iter().any(|r| r.peer_id == cred_a_peer_id);
let has_b = routes.iter().any(|r| r.peer_id == cred_b_peer_id);
let has_c = routes.iter().any(|r| r.peer_id == cred_c_peer_id);
println!("Admin routes: a={}, b={}, c={}", has_a, has_b, has_c);
has_a && has_b && has_c
},
Duration::from_secs(30),
)
.await;
// Wait for B and C to establish the candidate relay edge.
wait_for_condition(
|| async {
let peers_b = cred_b_inst.get_core_instance().connected_peers().await;
let peers_c = cred_c_inst.get_core_instance().connected_peers().await;
let b_connected_c = peers_b.contains(&cred_c_peer_id);
let c_connected_b = peers_c.contains(&cred_b_peer_id);
println!(
"P2P: B->C={}, C->B={}, allow_relay={}",
b_connected_c, c_connected_b, allow_relay
);
b_connected_c && c_connected_b
},
Duration::from_secs(30),
)
.await;
wait_route_cost(
&cred_a_inst,
cred_b_peer_id,
2,
Duration::from_secs(15),
"preference disabled on converged single-admin topology",
)
.await;
set_prefer_peer_relay(&admin_inst, true).await;
wait_route_cost(
&cred_a_inst,
cred_b_peer_id,
if allow_relay { 3 } else { 2 },
Duration::from_secs(20),
"enabling preference projects only authorized relay coverage",
)
.await;
if allow_relay {
set_prefer_peer_relay(&admin_inst, false).await;
wait_route_cost(
&cred_a_inst,
cred_b_peer_id,
2,
Duration::from_secs(20),
"disabling preference restores the admin edge",
)
.await;
set_prefer_peer_relay(&admin_inst, true).await;
wait_route_cost(
&cred_a_inst,
cred_b_peer_id,
3,
Duration::from_secs(20),
"re-enabling preference removes the admin edge again",
)
.await;
cred_a_inst.add_connector_url("tcp://10.1.1.4:11020".parse().unwrap());
wait_direct_peer(
&cred_a_inst,
cred_c_peer_id,
Duration::from_secs(10),
"credential A -> relay C",
)
.await;
}
let expected_next_hop = if allow_relay {
cred_c_peer_id
} else {
admin_peer_id
};
wait_route_next_hop(
&cred_a_inst,
cred_b_peer_id,
expected_next_hop,
Duration::from_secs(20),
"single-admin data path",
)
.await;
wait_route_next_hop(
&cred_b_inst,
cred_a_peer_id,
expected_next_hop,
Duration::from_secs(20),
"single-admin return data path",
)
.await;
wait_route_cost(
&admin_inst,
cred_b_peer_id,
1,
Duration::from_secs(10),
"admin retains its direct credential route",
)
.await;
let (expected_forwarder, forbidden_forwarder) = if allow_relay {
(&cred_c_inst, &admin_inst)
} else {
(&admin_inst, &cred_c_inst)
};
assert_ping_forwarded_by(
network.namespace("ns_c1"),
"10.144.144.3",
expected_forwarder,
&[forbidden_forwarder],
)
.await;
// Cleanup
drop_insts(vec![admin_inst, cred_a_inst, cred_b_inst, cred_c_inst]).await;
}
/// Both admins must project coverage from a credential relay even though the
/// credential only maintains one OSPF initiator session. The other admin gets
/// the relay's source row through admin-to-admin propagation. After projection
/// is proven, the source connects to the credential relay and sends data
/// through it. Routing must keep working after one admin stops.
///
/// Initially source, target, and relay each connect to both admins, while the
/// target also connects to the relay. After both admins project the relay's
/// coverage, the source connects directly to the relay for the data-path probe.
///
/// The source connects to both admins. Therefore its route to the target only
/// grows from two to three hops when both admins suppress their direct target
/// edge. Each admin must still retain its own direct fallback route.
#[tokio::test]
#[serial_test::serial]
async fn prefer_peer_relay_uses_forwarded_coverage_across_two_admins() {
let network = prepare_credential_network();
let process_runtime = CoreProcessRuntime::new();
let admin_a_config = create_admin_config(
"admin_a",
Some(network.namespace("ns_adm")),
"10.144.144.1",
"fd00::1/64",
);
let mut admin_a_inst =
Instance::new_with_process_runtime(admin_a_config, process_runtime.clone());
admin_a_inst.run().await.unwrap();
let admin_b_config = create_admin_config(
"admin_b",
Some(network.namespace("ns_c3")),
"10.144.144.4",
"fd00::4/64",
);
let mut admin_b_inst =
Instance::new_with_process_runtime(admin_b_config, process_runtime.clone());
admin_b_inst.run().await.unwrap();
admin_a_inst.add_connector_url("tcp://10.1.1.4:11010".parse().unwrap());
let admin_a_peer_id = admin_a_inst.peer_id();
let admin_b_peer_id = admin_b_inst.peer_id();
wait_direct_peer(
&admin_a_inst,
admin_b_peer_id,
Duration::from_secs(10),
"admin_a -> admin_b",
)
.await;
let (_, relay_secret) = generate_credential(
&admin_a_inst,
vec![],
true,
vec![],
Duration::from_secs(3600),
)
.await;
let (_, target_secret) = generate_credential(
&admin_a_inst,
vec![],
false,
vec![],
Duration::from_secs(3600),
)
.await;
let (_, source_secret) = generate_credential(
&admin_a_inst,
vec![],
false,
vec![],
Duration::from_secs(3600),
)
.await;
admin_a_inst
.get_global_ctx()
.issue_event(GlobalCtxEvent::CredentialChanged);
let network_name = admin_a_inst
.get_global_ctx()
.get_network_identity()
.network_name
.clone();
let relay_config = create_credential_config_from_secret(
network_name.clone(),
&relay_secret,
"relay",
Some(network.namespace("ns_c1")),
"10.144.144.2",
"fd00::2/64",
);
relay_config.set_listeners(vec!["tcp://0.0.0.0:11020".parse().unwrap()]);
disable_p2p(&relay_config);
let target_config = create_credential_config_from_secret(
network_name.clone(),
&target_secret,
"target",
Some(network.namespace("ns_c2")),
"10.144.144.3",
"fd00::3/64",
);
disable_p2p(&target_config);
let source_config = create_credential_config_from_secret(
network_name,
&source_secret,
"source",
Some(network.namespace("ns_c4")),
"10.144.144.5",
"fd00::5/64",
);
disable_p2p(&source_config);
let mut relay_inst = Instance::new_with_process_runtime(relay_config, process_runtime.clone());
let mut target_inst =
Instance::new_with_process_runtime(target_config, process_runtime.clone());
let mut source_inst =
Instance::new_with_process_runtime(source_config, process_runtime.clone());
relay_inst.run().await.unwrap();
target_inst.run().await.unwrap();
source_inst.run().await.unwrap();
let relay_listener_url: url::Url = "tcp://10.1.1.2:11020".parse().unwrap();
relay_inst.add_connector_url("tcp://10.1.1.1:11010".parse().unwrap());
relay_inst.add_connector_url("tcp://10.1.1.4:11010".parse().unwrap());
target_inst.add_connector_url("tcp://10.1.1.1:11010".parse().unwrap());
target_inst.add_connector_url("tcp://10.1.1.4:11010".parse().unwrap());
target_inst.add_connector_url(relay_listener_url.clone());
source_inst.add_connector_url("tcp://10.1.1.1:11010".parse().unwrap());
source_inst.add_connector_url("tcp://10.1.1.4:11010".parse().unwrap());
let relay_peer_id = relay_inst.peer_id();
let target_peer_id = target_inst.peer_id();
let source_peer_id = source_inst.peer_id();
for (admin, label) in [(&admin_a_inst, "admin_a"), (&admin_b_inst, "admin_b")] {
wait_direct_peer(
admin,
relay_peer_id,
Duration::from_secs(30),
&format!("{label} -> relay"),
)
.await;
wait_direct_peer(
admin,
target_peer_id,
Duration::from_secs(30),
&format!("{label} -> target"),
)
.await;
wait_direct_peer(
admin,
source_peer_id,
Duration::from_secs(30),
&format!("{label} -> source"),
)
.await;
}
wait_direct_peer(
&relay_inst,
admin_a_peer_id,
Duration::from_secs(10),
"relay -> admin_a",
)
.await;
wait_direct_peer(
&relay_inst,
admin_b_peer_id,
Duration::from_secs(10),
"relay -> admin_b",
)
.await;
wait_direct_peer(
&relay_inst,
target_peer_id,
Duration::from_secs(10),
"relay -> target",
)
.await;
wait_route_cost(
&source_inst,
target_peer_id,
2,
Duration::from_secs(30),
"source uses an admin direct target edge",
)
.await;
wait_ping_reachability(
network.namespace("ns_c4"),
"10.144.144.3",
true,
Duration::from_secs(10),
)
.await;
for admin in [&admin_a_inst, &admin_b_inst] {
set_prefer_peer_relay(admin, true).await;
}
wait_route_cost(
&source_inst,
target_peer_id,
3,
Duration::from_secs(30),
"both admins use credential relay coverage",
)
.await;
wait_route_cost(
&admin_a_inst,
target_peer_id,
1,
Duration::from_secs(10),
"admin_a retains direct fallback",
)
.await;
wait_route_cost(
&admin_b_inst,
target_peer_id,
1,
Duration::from_secs(10),
"admin_b retains direct fallback",
)
.await;
source_inst.add_connector_url(relay_listener_url.clone());
wait_direct_peer(
&source_inst,
relay_peer_id,
Duration::from_secs(10),
"source -> relay before admin shutdown",
)
.await;
wait_route_next_hop(
&source_inst,
target_peer_id,
relay_peer_id,
Duration::from_secs(20),
"dual-admin data path before one admin stops",
)
.await;
wait_route_next_hop(
&target_inst,
source_peer_id,
relay_peer_id,
Duration::from_secs(20),
"dual-admin return data path before one admin stops",
)
.await;
assert_ping_forwarded_by(
network.namespace("ns_c4"),
"10.144.144.3",
&relay_inst,
&[&admin_a_inst, &admin_b_inst],
)
.await;
// Equivalent admin candidates are ordered by peer id. Stop the first
// credential OSPF initiator so every credential must use the survivor.
let (stopped_admin, surviving_admin) = if admin_a_peer_id < admin_b_peer_id {
(admin_a_inst, admin_b_inst)
} else {
(admin_b_inst, admin_a_inst)
};
let stopped_admin_peer_id = stopped_admin.peer_id();
let surviving_admin_peer_id = surviving_admin.peer_id();
drop_insts(vec![stopped_admin]).await;
wait_for_condition(
|| async {
let peers = source_inst.get_core_instance().connected_peers().await;
!peers.contains(&stopped_admin_peer_id) && peers.contains(&surviving_admin_peer_id)
},
Duration::from_secs(15),
)
.await;
wait_route_next_hop(
&source_inst,
target_peer_id,
relay_peer_id,
Duration::from_secs(20),
"dual-admin data path remains on relay after one admin stops",
)
.await;
wait_route_next_hop(
&target_inst,
source_peer_id,
relay_peer_id,
Duration::from_secs(20),
"dual-admin return path remains on relay after one admin stops",
)
.await;
assert_ping_forwarded_by(
network.namespace("ns_c4"),
"10.144.144.3",
&relay_inst,
&[&surviving_admin],
)
.await;
wait_route_cost(
&surviving_admin,
target_peer_id,
1,
Duration::from_secs(10),
"surviving admin retains direct fallback",
)
.await;
// Removing and restoring the Peer Relay's target edge changes its
// source-owned connection row. The post-shutdown relay -> admin -> relay
// next-hop transition proves that fresh rows pass through the surviving
// admin; stale state cannot satisfy both observations.
disconnect_connector(&target_inst, &relay_listener_url, relay_peer_id).await;
wait_for_condition(
|| async {
!relay_inst
.get_core_instance()
.connected_peers()
.await
.contains(&target_peer_id)
},
Duration::from_secs(10),
)
.await;
wait_route_next_hop(
&source_inst,
target_peer_id,
surviving_admin_peer_id,
Duration::from_secs(30),
"relay edge removal restores the surviving admin data path",
)
.await;
wait_route_next_hop(
&target_inst,
source_peer_id,
surviving_admin_peer_id,
Duration::from_secs(30),
"relay edge removal restores the surviving admin return path",
)
.await;
assert_ping_forwarded_by(
network.namespace("ns_c4"),
"10.144.144.3",
&surviving_admin,
&[&relay_inst],
)
.await;
target_inst.add_connector_url(relay_listener_url.clone());
wait_direct_peer(
&target_inst,
relay_peer_id,
Duration::from_secs(10),
"target -> relay after admin shutdown",
)
.await;
wait_direct_peer(
&relay_inst,
target_peer_id,
Duration::from_secs(10),
"relay -> target after admin shutdown",
)
.await;
wait_route_next_hop(
&source_inst,
target_peer_id,
relay_peer_id,
Duration::from_secs(30),
"relay edge restoration restores the Peer Relay data path",
)
.await;
wait_route_next_hop(
&target_inst,
source_peer_id,
relay_peer_id,
Duration::from_secs(30),
"relay edge restoration restores the Peer Relay return path",
)
.await;
assert_ping_forwarded_by(
network.namespace("ns_c4"),
"10.144.144.3",
&relay_inst,
&[&surviving_admin],
)
.await;
drop_insts(vec![surviving_admin, relay_inst, target_inst, source_inst]).await;
}
/// Test 2: Two credential nodes connect to same admin
/// Topology: Admin ← Credential_A, Admin ← Credential_B
/// Verifies that multiple credential nodes can connect to the same admin
#[tokio::test]
#[serial_test::serial]
async fn credential_two_credentials_communicate_tcp() {
let network = prepare_credential_network();
let process_runtime = CoreProcessRuntime::new();
// Create admin node
let admin_config = create_admin_config(
"admin",
Some(network.namespace("ns_adm")),
"10.144.144.1",
"fd00::1/64",
);
let mut admin_inst = Instance::new_with_process_runtime(admin_config, process_runtime.clone());
admin_inst.run().await.unwrap();
// Create credential1 on ns_c1
let cred1_config = create_credential_config(
&admin_inst,
"cred1",
Some(network.namespace("ns_c1")),
"10.144.144.2",
"fd00::2/64",
)
.await;
let mut cred1_inst = Instance::new_with_process_runtime(cred1_config, process_runtime.clone());
cred1_inst.run().await.unwrap();
// Create credential2 on ns_c2
let cred2_config = create_credential_config(
&admin_inst,
"cred2",
Some(network.namespace("ns_c2")),
"10.144.144.3",
"fd00::3/64",
)
.await;
let mut cred2_inst = Instance::new_with_process_runtime(cred2_config, process_runtime.clone());
cred2_inst.run().await.unwrap();
// Both credentials connect to admin
cred1_inst.add_connector_url("tcp://10.1.1.1:11010".parse().unwrap());
cred2_inst.add_connector_url("tcp://10.1.1.1:11010".parse().unwrap());
let cred1_peer_id = cred1_inst.peer_id();
let cred2_peer_id = cred2_inst.peer_id();
// Wait for both credentials to appear in admin's route table
wait_for_condition(
|| async {
let routes = admin_inst.get_core_instance().route_snapshots().await;
routes.iter().any(|r| r.peer_id == cred1_peer_id)
&& routes.iter().any(|r| r.peer_id == cred2_peer_id)
},
Duration::from_secs(10),
)
.await;
// Verify admin can ping both credentials
wait_for_condition(
|| async { ping_test(network.namespace("ns_adm"), "10.144.144.2", None).await },
Duration::from_secs(10),
)
.await;
wait_for_condition(
|| async { ping_test(network.namespace("ns_adm"), "10.144.144.3", None).await },
Duration::from_secs(10),
)
.await;
drop_insts(vec![admin_inst, cred1_inst, cred2_inst]).await;
}
/// Test 3: Credential revocation removes credential from route table
/// Topology: Admin ← Credential
/// Verifies that when credential is revoked, it's removed from admin's route table
#[tokio::test]
#[serial_test::serial]
async fn credential_revocation_propagates() {
let network = prepare_credential_network();
let process_runtime = CoreProcessRuntime::new();
// Create admin on ns_adm (10.1.1.1)
let admin_config = create_admin_config(
"admin",
Some(network.namespace("ns_adm")),
"10.144.144.1",
"fd00::1/64",
);
let mut admin_inst = Instance::new_with_process_runtime(admin_config, process_runtime.clone());
admin_inst.run().await.unwrap();
// Generate credential on admin
let (cred_id, cred_secret) = generate_credential(
&admin_inst,
vec![],
false,
vec![],
Duration::from_secs(3600),
)
.await;
// Create credential node
let cred_config = {
use base64::Engine as _;
let privkey_bytes: [u8; 32] = base64::prelude::BASE64_STANDARD
.decode(&cred_secret)
.unwrap()
.try_into()
.unwrap();
let private = x25519_dalek::StaticSecret::from(privkey_bytes);
let config = TomlConfigLoader::default();
config.set_inst_name("cred".to_string());
config.set_netns(Some(network.namespace("ns_c1").to_string()));
config.set_ipv4(Some("10.144.144.2".parse().unwrap()));
config.set_ipv6(Some("fd00::2/64".parse().unwrap()));
config.set_listeners(vec![]);
config.set_network_identity(NetworkIdentity::new_credential(
admin_inst
.get_global_ctx()
.get_network_identity()
.network_name
.clone(),
));
config.set_secure_mode(Some(generate_secure_mode_config_with_key(&private)));
config
};
let mut cred_inst = Instance::new_with_process_runtime(cred_config, process_runtime.clone());
cred_inst.run().await.unwrap();
// Credential connects to admin
cred_inst.add_connector_url("tcp://10.1.1.1:11010".parse().unwrap());
let cred_peer_id = cred_inst.peer_id();
// Wait for credential to appear in admin's route table
wait_for_condition(
|| async {
admin_inst
.get_core_instance()
.route_snapshots()
.await
.iter()
.any(|r| r.peer_id == cred_peer_id)
},
Duration::from_secs(10),
)
.await;
// Verify connectivity before revocation
wait_for_condition(
|| async { ping_test(network.namespace("ns_adm"), "10.144.144.2", None).await },
Duration::from_secs(10),
)
.await;
// Revoke the credential
assert!(
admin_inst
.get_core_instance()
.revoke_credential(&cred_id)
.unwrap(),
"Credential should be revoked successfully"
);
// Trigger OSPF sync
admin_inst
.get_global_ctx()
.issue_event(GlobalCtxEvent::CredentialChanged);
// Wait for credential to disappear from admin's route table
wait_for_condition(
|| async {
!admin_inst
.get_core_instance()
.route_snapshots()
.await
.iter()
.any(|r| r.peer_id == cred_peer_id)
},
Duration::from_secs(15),
)
.await;
wait_for_condition(
|| async { !ping_test(network.namespace("ns_adm"), "10.144.144.2", None).await },
Duration::from_secs(10),
)
.await;
wait_for_condition(
|| async { !ping_test(network.namespace("ns_c1"), "10.144.144.1", None).await },
Duration::from_secs(10),
)
.await;
drop_insts(vec![admin_inst, cred_inst]).await;
}
/// Test: A non-reusable credential only allows one peer at a time.
#[tokio::test]
#[serial_test::serial]
async fn credential_non_reusable_allows_only_one_peer() {
let network = prepare_credential_network();
let process_runtime = CoreProcessRuntime::new();
let admin_config = create_admin_config(
"admin",
Some(network.namespace("ns_adm")),
"10.144.144.1",
"fd00::1/64",
);
let mut admin_inst = Instance::new_with_process_runtime(admin_config, process_runtime.clone());
admin_inst.run().await.unwrap();
let (_cred_id, cred_secret) = generate_credential_with_options(
&admin_inst,
vec![],
false,
vec![],
Duration::from_secs(3600),
None,
false,
)
.await;
let network_name = admin_inst
.get_global_ctx()
.get_network_identity()
.network_name
.clone();
let cred1_config = create_credential_config_from_secret(
network_name.clone(),
&cred_secret,
"cred1_single",
Some(network.namespace("ns_c1")),
"10.144.144.2",
"fd00::2/64",
);
let cred2_config = create_credential_config_from_secret(
network_name,
&cred_secret,
"cred2_single",
Some(network.namespace("ns_c2")),
"10.144.144.3",
"fd00::3/64",
);
let mut cred1_inst = Some(Instance::new_with_process_runtime(
cred1_config,
process_runtime.clone(),
));
cred1_inst.as_mut().unwrap().run().await.unwrap();
cred1_inst
.as_ref()
.unwrap()
.add_connector_url("tcp://10.1.1.1:11010".parse().unwrap());
let cred1_peer_id = cred1_inst.as_ref().unwrap().peer_id();
wait_for_condition(
|| async {
admin_inst
.get_core_instance()
.route_snapshots()
.await
.iter()
.any(|r| r.peer_id == cred1_peer_id)
},
Duration::from_secs(10),
)
.await;
wait_ping_reachability(
network.namespace("ns_adm"),
"10.144.144.2",
true,
Duration::from_secs(10),
)
.await;
let mut cred2_inst = Some(Instance::new_with_process_runtime(
cred2_config,
process_runtime.clone(),
));
cred2_inst.as_mut().unwrap().run().await.unwrap();
cred2_inst
.as_ref()
.unwrap()
.add_connector_url("tcp://10.1.1.1:11010".parse().unwrap());
let cred2_peer_id = cred2_inst.as_ref().unwrap().peer_id();
tokio::time::sleep(Duration::from_secs(3)).await;
// The non-reusable credential owner is elected by lowest peer_id, so either cred1 or cred2
// may win. Determine the winner and loser dynamically.
let admin_routes = admin_inst.get_core_instance().route_snapshots().await;
let (winner_peer_id, winner_ip, winner_inst, loser_peer_id, loser_ip, loser_inst) =
if admin_routes.iter().any(|r| r.peer_id == cred1_peer_id) {
(
cred1_peer_id,
"10.144.144.2",
&mut cred1_inst,
cred2_peer_id,
"10.144.144.3",
&mut cred2_inst,
)
} else if admin_routes.iter().any(|r| r.peer_id == cred2_peer_id) {
(
cred2_peer_id,
"10.144.144.3",
&mut cred2_inst,
cred1_peer_id,
"10.144.144.2",
&mut cred1_inst,
)
} else {
panic!(
"neither credential peer is present in routes: {:?}",
admin_routes.iter().map(|r| r.peer_id).collect::<Vec<_>>()
);
};
for _ in 0..5 {
let admin_routes = admin_inst.get_core_instance().route_snapshots().await;
assert!(
admin_routes.iter().any(|r| r.peer_id == winner_peer_id),
"winning credential peer should remain present: {:?}",
admin_routes.iter().map(|r| r.peer_id).collect::<Vec<_>>()
);
assert!(
ping_test(network.namespace("ns_adm"), winner_ip, None).await,
"admin should still reach the winning credential peer"
);
tokio::time::sleep(Duration::from_secs(1)).await;
}
for _ in 0..5 {
let admin_routes = admin_inst.get_core_instance().route_snapshots().await;
assert!(
!admin_routes.iter().any(|r| r.peer_id == loser_peer_id),
"losing credential peer should not appear in routes: {:?}",
admin_routes.iter().map(|r| r.peer_id).collect::<Vec<_>>()
);
assert!(
!ping_test(network.namespace("ns_adm"), loser_ip, None).await,
"admin should not reach the losing credential peer"
);
tokio::time::sleep(Duration::from_secs(1)).await;
}
drop_insts(vec![winner_inst.take().unwrap()]).await;
wait_for_condition(
|| async {
let routes = admin_inst.get_core_instance().route_snapshots().await;
!routes.iter().any(|r| r.peer_id == winner_peer_id)
&& routes.iter().any(|r| r.peer_id == loser_peer_id)
},
Duration::from_secs(20),
)
.await;
wait_ping_reachability(
network.namespace("ns_adm"),
loser_ip,
true,
Duration::from_secs(20),
)
.await;
drop_insts(vec![admin_inst, loser_inst.take().unwrap()]).await;
}
/// Test 4: Unknown credential (not in trusted list) is rejected
/// Topology: Admin
/// Verifies that credential nodes with unknown/random keys cannot connect
#[tokio::test]
#[serial_test::serial]
async fn credential_unknown_rejected() {
let network = prepare_credential_network();
let process_runtime = CoreProcessRuntime::new();
// Create admin node
let admin_config = create_admin_config(
"admin",
Some(network.namespace("ns_adm")),
"10.144.144.1",
"fd00::1/64",
);
let mut admin_inst = Instance::new_with_process_runtime(admin_config, process_runtime.clone());
admin_inst.run().await.unwrap();
// Create credential node with random key (not generated by admin)
let random_private = x25519_dalek::StaticSecret::random_from_rng(rand::rngs::OsRng);
let cred_config = {
let config = TomlConfigLoader::default();
config.set_inst_name("cred".to_string());
config.set_netns(Some(network.namespace("ns_c1").to_string()));
config.set_ipv4(Some("10.144.144.2".parse().unwrap()));
config.set_ipv6(Some("fd00::2/64".parse().unwrap()));
config.set_listeners(vec![]);
config.set_network_identity(NetworkIdentity::new_credential(
admin_inst
.get_global_ctx()
.get_network_identity()
.network_name
.clone(),
));
config.set_secure_mode(Some(generate_secure_mode_config_with_key(&random_private)));
config
};
let mut cred_inst = Instance::new_with_process_runtime(cred_config, process_runtime.clone());
cred_inst.run().await.unwrap();
// Attempt to connect to admin
cred_inst.add_connector_url("tcp://10.1.1.1:11010".parse().unwrap());
let cred_peer_id = cred_inst.peer_id();
// Wait a bit for connection attempt
tokio::time::sleep(Duration::from_secs(5)).await;
// Verify credential does NOT appear in admin's route table
let routes = admin_inst.get_core_instance().route_snapshots().await;
assert!(
!routes.iter().any(|r| r.peer_id == cred_peer_id),
"Unknown credential node should NOT appear in admin's route table"
);
// Verify no connectivity
let ping_result = ping_test(network.namespace("ns_adm"), "10.144.144.2", None).await;
assert!(
!ping_result,
"Should NOT be able to ping unknown credential node"
);
drop_insts(vec![admin_inst, cred_inst]).await;
}
/// Regression test: an unknown credential must still be rejected when it first connects via a
/// shared node. If this fails, the shared path is incorrectly admitting the node into the target
/// network's route domain.
#[rstest::rstest]
#[tokio::test]
#[serial_test::serial]
async fn credential_unknown_via_shared_rejected(#[values(true, false)] test_revoke: bool) {
let network = prepare_credential_network();
let process_runtime = CoreProcessRuntime::new();
let admin_a_config = create_admin_config(
"admin_a",
Some(network.namespace("ns_adm")),
"10.144.144.1",
"fd00::1/64",
);
let mut admin_a_inst =
Instance::new_with_process_runtime(admin_a_config, process_runtime.clone());
admin_a_inst.run().await.unwrap();
let shared_b_config = create_shared_config(
"shared_b",
Some(network.namespace("ns_c1")),
"10.144.144.2",
"fd00::2/64",
);
let mut shared_b_inst =
Instance::new_with_process_runtime(shared_b_config, process_runtime.clone());
shared_b_inst.run().await.unwrap();
let admin_c_config = create_admin_config(
"admin_c",
Some(network.namespace("ns_c3")),
"10.144.144.4",
"fd00::4/64",
);
let mut admin_c_inst =
Instance::new_with_process_runtime(admin_c_config, process_runtime.clone());
admin_c_inst.run().await.unwrap();
admin_a_inst.add_connector_url("tcp://10.1.1.2:11010".parse().unwrap());
admin_c_inst.add_connector_url("tcp://10.1.1.2:11010".parse().unwrap());
let admin_c_peer_id = admin_c_inst.peer_id();
wait_for_condition(
|| async {
let a_routes = admin_a_inst.get_core_instance().route_snapshots().await;
let c_routes = admin_c_inst.get_core_instance().route_snapshots().await;
a_routes.iter().any(|r| r.peer_id == admin_c_peer_id)
|| c_routes.iter().any(|r| r.peer_id == admin_a_inst.peer_id())
},
Duration::from_secs(10),
)
.await;
let (credential_config, credential_id) = if test_revoke {
let (config, cred_id) = create_generated_credential_config(
&admin_a_inst,
"cred_d",
Some(network.namespace("ns_c2")),
"10.144.144.5",
"fd00::5/64",
)
.await;
(config, Some(cred_id))
} else {
(
create_unknown_credential_config(
admin_a_inst
.get_global_ctx()
.get_network_identity()
.network_name
.clone(),
"unknown_d",
Some(network.namespace("ns_c2")),
"10.144.144.5",
"fd00::5/64",
),
None,
)
};
let mut unknown_inst =
Instance::new_with_process_runtime(credential_config, process_runtime.clone());
unknown_inst.run().await.unwrap();
unknown_inst.add_connector_url("tcp://10.1.1.2:11010".parse().unwrap());
let unknown_peer_id = unknown_inst.peer_id();
println!("unknown_peer_id: {:?}", unknown_peer_id);
if test_revoke {
wait_route_presence_on_admins(
&admin_a_inst,
&admin_c_inst,
unknown_peer_id,
true,
Duration::from_secs(30),
)
.await;
wait_ping_reachability(
network.namespace("ns_adm"),
"10.144.144.5",
true,
Duration::from_secs(20),
)
.await;
wait_ping_reachability(
network.namespace("ns_c3"),
"10.144.144.5",
true,
Duration::from_secs(20),
)
.await;
assert!(
admin_a_inst
.get_core_instance()
.revoke_credential(credential_id.as_ref().unwrap())
.unwrap(),
"credential should be revoked successfully"
);
admin_a_inst
.get_global_ctx()
.issue_event(GlobalCtxEvent::CredentialChanged);
wait_route_presence_on_admins(
&admin_a_inst,
&admin_c_inst,
unknown_peer_id,
false,
Duration::from_secs(30),
)
.await;
wait_ping_reachability(
network.namespace("ns_adm"),
"10.144.144.5",
false,
Duration::from_secs(5),
)
.await;
wait_ping_reachability(
network.namespace("ns_c3"),
"10.144.144.5",
false,
Duration::from_secs(5),
)
.await;
unknown_inst.add_connector_url("tcp://10.1.1.2:11010".parse().unwrap());
assert_shared_visibility_stable(
&network,
&admin_a_inst,
&admin_c_inst,
unknown_peer_id,
"10.144.144.5",
false,
"revoked credential",
)
.await;
} else {
assert_shared_visibility_stable(
&network,
&admin_a_inst,
&admin_c_inst,
unknown_peer_id,
"10.144.144.5",
false,
"unknown credential",
)
.await;
}
println!("drop all");
drop_insts(vec![
admin_a_inst,
shared_b_inst,
admin_c_inst,
unknown_inst,
])
.await;
}
#[rstest::rstest]
#[tokio::test]
#[serial_test::serial]
async fn credential_admin_shared_admin_credential_connectivity(
#[values(true, false)] connect_to_admin: bool,
) {
let network = prepare_credential_network();
let process_runtime = CoreProcessRuntime::new();
// Keep traffic on the shared/admin relay path; direct P2P would bypass it.
// 10.1.1.1
let admin_a_config = create_admin_config(
"admin_a",
Some(network.namespace("ns_adm")),
"10.144.144.1",
"fd00::1/64",
);
disable_p2p(&admin_a_config);
let mut admin_a_inst =
Instance::new_with_process_runtime(admin_a_config, process_runtime.clone());
admin_a_inst.run().await.unwrap();
// 10.1.1.2
let shared_b_config = create_shared_config(
"shared_b",
Some(network.namespace("ns_c1")),
"10.144.144.2",
"fd00::2/64",
);
disable_p2p(&shared_b_config);
let mut shared_b_inst =
Instance::new_with_process_runtime(shared_b_config, process_runtime.clone());
shared_b_inst.run().await.unwrap();
// 10.1.1.4
let admin_c_config = create_admin_config(
"admin_c",
Some(network.namespace("ns_c3")),
"10.144.144.4",
"fd00::4/64",
);
disable_p2p(&admin_c_config);
let mut admin_c_inst =
Instance::new_with_process_runtime(admin_c_config, process_runtime.clone());
admin_c_inst.run().await.unwrap();
admin_a_inst.add_connector_url("tcp://10.1.1.2:11010".parse().unwrap());
admin_c_inst.add_connector_url("tcp://10.1.1.2:11010".parse().unwrap());
// print all peer ids
println!("admin_a_peer_id: {:?}", admin_a_inst.peer_id());
println!("shared_b_peer_id: {:?}", shared_b_inst.peer_id());
println!("admin_c_peer_id: {:?}", admin_c_inst.peer_id());
let admin_c_peer_id = admin_c_inst.peer_id();
wait_for_condition(
|| async {
let a_routes = admin_a_inst.get_core_instance().route_snapshots().await;
let c_routes = admin_c_inst.get_core_instance().route_snapshots().await;
println!(
"bootstrap routes: a={:?} c={:?}",
a_routes.iter().map(|r| r.peer_id).collect::<Vec<_>>(),
c_routes.iter().map(|r| r.peer_id).collect::<Vec<_>>()
);
a_routes.iter().any(|r| r.peer_id == admin_c_peer_id)
|| c_routes.iter().any(|r| r.peer_id == admin_a_inst.peer_id())
},
Duration::from_secs(3),
)
.await;
let cred_d_config = create_credential_config(
&admin_a_inst,
"cred_d",
Some(network.namespace("ns_c2")),
"10.144.144.5",
"fd00::5/64",
)
.await;
admin_a_inst
.get_global_ctx()
.issue_event(GlobalCtxEvent::CredentialChanged);
disable_p2p(&cred_d_config);
let mut cred_d_inst =
Instance::new_with_process_runtime(cred_d_config, process_runtime.clone());
cred_d_inst.run().await.unwrap();
let cred_d_peer_id = cred_d_inst.peer_id();
cred_d_inst.add_connector_url(if !connect_to_admin {
// connect to shared node
"tcp://10.1.1.2:11010".parse().unwrap()
} else {
// connect to admin node
"tcp://10.1.1.4:11010".parse().unwrap()
});
// print all peer ids
println!("cred_d_peer_id: {:?}", cred_d_peer_id);
wait_for_condition(
|| async {
admin_c_inst
.get_core_instance()
.route_snapshots()
.await
.iter()
.any(|r| r.peer_id == cred_d_peer_id)
},
Duration::from_secs(60),
)
.await;
wait_for_condition(
|| async { ping_test(network.namespace("ns_c3"), "10.144.144.5", None).await },
Duration::from_secs(15),
)
.await;
wait_for_condition(
|| async { ping_test(network.namespace("ns_adm"), "10.144.144.5", None).await },
Duration::from_secs(15),
)
.await;
wait_for_condition(
|| async { ping_test(network.namespace("ns_c2"), "10.144.144.4", None).await },
Duration::from_secs(15),
)
.await;
drop_insts(vec![admin_a_inst, shared_b_inst, admin_c_inst, cred_d_inst]).await;
}
#[tokio::test]
#[serial_test::serial]
async fn credential_non_reusable_across_two_admins_allows_only_one_peer() {
let network = prepare_credential_network();
let process_runtime = CoreProcessRuntime::new();
let admin_a_config = create_admin_config(
"admin_a",
Some(network.namespace("ns_adm")),
"10.144.144.1",
"fd00::1/64",
);
let mut admin_a_inst =
Instance::new_with_process_runtime(admin_a_config, process_runtime.clone());
admin_a_inst.run().await.unwrap();
let shared_b_config = create_shared_config(
"shared_b",
Some(network.namespace("ns_c1")),
"10.144.144.2",
"fd00::2/64",
);
let mut shared_b_inst =
Instance::new_with_process_runtime(shared_b_config, process_runtime.clone());
shared_b_inst.run().await.unwrap();
let admin_c_config = create_admin_config(
"admin_c",
Some(network.namespace("ns_c3")),
"10.144.144.4",
"fd00::4/64",
);
let mut admin_c_inst =
Instance::new_with_process_runtime(admin_c_config, process_runtime.clone());
admin_c_inst.run().await.unwrap();
admin_a_inst.add_connector_url("tcp://10.1.1.2:11010".parse().unwrap());
admin_c_inst.add_connector_url("tcp://10.1.1.2:11010".parse().unwrap());
let admin_c_peer_id = admin_c_inst.peer_id();
wait_for_condition(
|| async {
let a_routes = admin_a_inst.get_core_instance().route_snapshots().await;
let c_routes = admin_c_inst.get_core_instance().route_snapshots().await;
a_routes.iter().any(|r| r.peer_id == admin_c_peer_id)
&& c_routes.iter().any(|r| r.peer_id == admin_a_inst.peer_id())
},
Duration::from_secs(20),
)
.await;
let (_cred_id, cred_secret) = generate_credential_with_options(
&admin_a_inst,
vec![],
false,
vec![],
Duration::from_secs(3600),
None,
false,
)
.await;
admin_a_inst
.get_global_ctx()
.issue_event(GlobalCtxEvent::CredentialChanged);
let network_name = admin_a_inst
.get_global_ctx()
.get_network_identity()
.network_name
.clone();
let cred_left_config = create_credential_config_from_secret(
network_name.clone(),
&cred_secret,
"cred_left",
Some(network.namespace("ns_c2")),
"10.144.144.5",
"fd00::5/64",
);
let cred_right_config = create_credential_config_from_secret(
network_name,
&cred_secret,
"cred_right",
Some(network.namespace("ns_c4")),
"10.144.144.6",
"fd00::6/64",
);
let mut cred_left_inst = Some(Instance::new_with_process_runtime(
cred_left_config,
process_runtime.clone(),
));
cred_left_inst.as_mut().unwrap().run().await.unwrap();
cred_left_inst
.as_ref()
.unwrap()
.add_connector_url("tcp://10.1.1.1:11010".parse().unwrap());
let mut cred_right_inst = Some(Instance::new_with_process_runtime(
cred_right_config,
process_runtime.clone(),
));
cred_right_inst.as_mut().unwrap().run().await.unwrap();
cred_right_inst
.as_ref()
.unwrap()
.add_connector_url("tcp://10.1.1.4:11010".parse().unwrap());
let cred_left_peer_id = cred_left_inst.as_ref().unwrap().peer_id();
let cred_right_peer_id = cred_right_inst.as_ref().unwrap().peer_id();
let winner_peer_id = wait_stable_single_visible_peer_on_admins(
&admin_a_inst,
&admin_c_inst,
cred_left_peer_id,
cred_right_peer_id,
Duration::from_secs(60),
)
.await;
let (winner_peer_id, winner_ip, loser_peer_id, loser_ip) =
if winner_peer_id == cred_left_peer_id {
(
cred_left_peer_id,
"10.144.144.5",
cred_right_peer_id,
"10.144.144.6",
)
} else {
(
cred_right_peer_id,
"10.144.144.6",
cred_left_peer_id,
"10.144.144.5",
)
};
wait_ping_reachability(
network.namespace("ns_adm"),
winner_ip,
true,
Duration::from_secs(20),
)
.await;
wait_ping_reachability(
network.namespace("ns_c3"),
winner_ip,
true,
Duration::from_secs(20),
)
.await;
wait_ping_reachability(
network.namespace("ns_adm"),
loser_ip,
false,
Duration::from_secs(5),
)
.await;
wait_ping_reachability(
network.namespace("ns_c3"),
loser_ip,
false,
Duration::from_secs(5),
)
.await;
assert_shared_visibility_stable(
&network,
&admin_a_inst,
&admin_c_inst,
winner_peer_id,
winner_ip,
true,
"winning credential",
)
.await;
assert_shared_visibility_stable(
&network,
&admin_a_inst,
&admin_c_inst,
loser_peer_id,
loser_ip,
false,
"losing credential",
)
.await;
if winner_peer_id == cred_left_peer_id {
drop_insts(vec![cred_left_inst.take().unwrap()]).await;
} else {
drop_insts(vec![cred_right_inst.take().unwrap()]).await;
}
wait_stable_failover_visibility_on_admins(
&admin_a_inst,
&admin_c_inst,
loser_peer_id,
winner_peer_id,
Duration::from_secs(60),
)
.await;
wait_ping_reachability(
network.namespace("ns_adm"),
loser_ip,
true,
Duration::from_secs(20),
)
.await;
wait_ping_reachability(
network.namespace("ns_c3"),
loser_ip,
true,
Duration::from_secs(20),
)
.await;
wait_ping_reachability(
network.namespace("ns_adm"),
winner_ip,
false,
Duration::from_secs(5),
)
.await;
wait_ping_reachability(
network.namespace("ns_c3"),
winner_ip,
false,
Duration::from_secs(5),
)
.await;
assert_shared_visibility_stable(
&network,
&admin_a_inst,
&admin_c_inst,
loser_peer_id,
loser_ip,
true,
"failover credential",
)
.await;
assert_shared_visibility_stable(
&network,
&admin_a_inst,
&admin_c_inst,
winner_peer_id,
winner_ip,
false,
"removed winning credential",
)
.await;
if winner_peer_id == cred_left_peer_id {
drop_insts(vec![
admin_a_inst,
shared_b_inst,
admin_c_inst,
cred_right_inst.take().unwrap(),
])
.await;
} else {
drop_insts(vec![
admin_a_inst,
shared_b_inst,
admin_c_inst,
cred_left_inst.take().unwrap(),
])
.await;
}
}
Reference in New Issue View Git Blame Copy Permalink