From 7c8ca07812daeda3eec90bbb86f19c3f89cc0ee8 Mon Sep 17 00:00:00 2001 From: KKRainbow <5665404+KKRainbow@users.noreply.github.com> Date: Tue, 13 Oct 2026 23:53:35 +0800 Subject: [PATCH] refactor(tun): consolidate shared NIC configuration and tests (#2690) * refactor(tun): consolidate shared NIC configuration and tests Separate member claims from OS deltas so mobile admission updates do not compute unused diffs. Reuse effective configurations for desktop address, route, and MTU updates, and share the platform claims paths. Remove redundant runtime dispatch and test-only public accessors while preserving registration cleanup and platform device creation ordering. Move configuration and lifecycle tests into focused modules. Reuse setup and table-driven cases, and verify route replacement order, source hints, stale registrations, and failed cleanup through behavior. Validation: shared NIC unit and Docker integration tests, mobile claim selection, TUN and public IPv6 tests, full-workspace clippy, Android arm64 library check, and formatting of the changed files. * docs(tun): describe shared Ring dispatch lifecycle Document direct member Ring polling, mobile descriptor replacement, and cooperative scheduling in the shared dispatcher. The dispatcher implementation is already provided by the base branch in #2689. * test(core): remove completion race from timeout budget test Use a pending future to exercise stage-specific timeout errors. A delayed CI runner can observe the old 20 ms sleep and 1 ms timeout as ready together. Tokio polls the inner future first, returning Ok instead of the error expected by the test. Keep the timeout assertion without a competing completion timer. Production timeout behavior is unchanged. --- docs/core-architecture.md | 6 + easytier-core/src/connectivity/manual/mod.rs | 15 +- easytier/src/instance/shared_virtual_nic.rs | 1140 +---------------- .../src/instance/shared_virtual_nic/ifcfg.rs | 272 ++++ .../shared_virtual_nic/ifcfg/tests.rs | 128 ++ .../src/instance/shared_virtual_nic/tests.rs | 322 +++++ 6 files changed, 796 insertions(+), 1087 deletions(-) create mode 100644 easytier/src/instance/shared_virtual_nic/ifcfg.rs create mode 100644 easytier/src/instance/shared_virtual_nic/ifcfg/tests.rs create mode 100644 easytier/src/instance/shared_virtual_nic/tests.rs diff --git a/docs/core-architecture.md b/docs/core-architecture.md index 6459f204..d32ee1a6 100644 --- a/docs/core-architecture.md +++ b/docs/core-architecture.md @@ -445,6 +445,12 @@ and the last member's cleanup stops the dispatcher and releases the desktop device. A process-lifetime runtime does not extend the device's lifetime. Unnamed desktop TUNs and instance-local work retain their existing executors. +Each member connects through one bounded Ring Tunnel. The shared dispatcher +polls member streams in rotation and writes directly to their sinks, with no +per-member forwarding tasks or intermediate packet queues. Desktop and mobile +use one dispatch loop; mobile descriptor replacement is a control message. +The loop consumes Tokio's cooperative budget even when packets are dropped. + Only the member running the Magic DNS resolver claims its shared fake-IP route. The claim is removed before resolver shutdown and acquired by the next serving member after takeover. Client-only runners do not claim that route. diff --git a/easytier-core/src/connectivity/manual/mod.rs b/easytier-core/src/connectivity/manual/mod.rs index 40602164..b3fc6677 100644 --- a/easytier-core/src/connectivity/manual/mod.rs +++ b/easytier-core/src/connectivity/manual/mod.rs @@ -1381,13 +1381,14 @@ mod tests { #[tokio::test] async fn timeout_budget_reports_the_active_stage() { - let error = - with_timeout_budget("connect", Instant::now(), Duration::from_millis(1), async { - crate::foundation::time::sleep(Duration::from_millis(20)).await; - Ok::<(), anyhow::Error>(()) - }) - .await - .unwrap_err(); + let error = with_timeout_budget( + "connect", + Instant::now(), + Duration::from_millis(1), + std::future::pending::>(), + ) + .await + .unwrap_err(); assert!(error.to_string().contains("connect timeout after")); } diff --git a/easytier/src/instance/shared_virtual_nic.rs b/easytier/src/instance/shared_virtual_nic.rs index 1e54beeb..f8041100 100644 --- a/easytier/src/instance/shared_virtual_nic.rs +++ b/easytier/src/instance/shared_virtual_nic.rs @@ -7,12 +7,11 @@ use std::{ }, }; -use cidr::{Ipv4Cidr, Ipv4Inet, Ipv6Cidr, Ipv6Inet}; +use cidr::{Ipv4Inet, Ipv6Inet}; use tokio::{ runtime::{Builder, Runtime}, sync::{Mutex, Notify}, }; -use tokio_util::task::AbortOnDropHandle; use crate::common::error::Error; use easytier_core::tunnel::Tunnel; @@ -23,6 +22,13 @@ use super::virtual_nic::{VirtualNic, VirtualNicConfig}; use crate::common::ifcfg::IfConfiger; mod dispatcher; +mod ifcfg; + +pub(crate) use ifcfg::SharedIfConfig; +#[cfg(any(mobile, test))] +pub(crate) use ifcfg::SharedIfConfigConflict; +use ifcfg::SharedIfConfigDelta; +pub use ifcfg::{SharedIfConfigClaims, SharedIpv4Route, SharedIpv6Route}; use dispatcher::{SharedVirtualNicDispatcher, SharedVirtualNicMemberTunnelTable}; @@ -40,297 +46,6 @@ static SHARED_VIRTUAL_NIC_RUNTIME: LazyLock = LazyLock::new(|| { .expect("failed to build shared virtual nic runtime") }); -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub struct SharedIpv4Route { - pub address: Ipv4Addr, - pub prefix: u8, - pub cost: Option, -} - -impl SharedIpv4Route { - pub fn new(address: Ipv4Addr, prefix: u8, cost: Option) -> Self { - Self { - address, - prefix, - cost, - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub struct SharedIpv6Route { - pub address: Ipv6Addr, - pub prefix: u8, - pub cost: Option, -} - -impl SharedIpv6Route { - pub fn new(address: Ipv6Addr, prefix: u8, cost: Option) -> Self { - Self { - address, - prefix, - cost, - } - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct SharedIfConfigClaims { - pub ipv4_addresses: BTreeSet, - pub ipv6_addresses: BTreeSet, - pub ipv4_routes: BTreeSet, - pub ipv6_routes: BTreeSet, - pub mtu: Option, -} - -impl SharedIfConfigClaims { - fn ipv4_destinations(&self) -> impl Iterator + '_ { - self.ipv4_addresses - .iter() - .map(|ip| (ip.network(), false)) - .chain( - self.ipv4_routes - .iter() - .filter_map(|route| Ipv4Inet::new(route.address, route.prefix).ok()) - .map(|route| (route.network(), true)), - ) - } - - fn ipv6_destinations(&self) -> impl Iterator + '_ { - self.ipv6_addresses.iter().map(Ipv6Inet::network).chain( - self.ipv6_routes - .iter() - .filter_map(|route| Ipv6Inet::new(route.address, route.prefix).ok()) - .map(|route| route.network()), - ) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct OwnedItemDelta { - pub added: BTreeSet, - pub removed: BTreeSet, -} - -impl Default for OwnedItemDelta { - fn default() -> Self { - Self { - added: BTreeSet::new(), - removed: BTreeSet::new(), - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct SharedMtuChange { - pub old: Option, - pub new: Option, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct SharedIfConfigDelta { - pub ipv4_addresses: OwnedItemDelta, - pub ipv6_addresses: OwnedItemDelta, - pub ipv4_routes: OwnedItemDelta, - pub ipv6_routes: OwnedItemDelta, - pub mtu: Option, -} - -impl SharedIfConfigDelta { - fn between(old: &EffectiveIfConfig, new: &EffectiveIfConfig) -> Self { - Self { - ipv4_addresses: item_delta(&old.ipv4_addresses, &new.ipv4_addresses), - ipv6_addresses: item_delta(&old.ipv6_addresses, &new.ipv6_addresses), - ipv4_routes: item_delta(&old.ipv4_routes, &new.ipv4_routes), - ipv6_routes: item_delta(&old.ipv6_routes, &new.ipv6_routes), - mtu: mtu_delta(old.effective_mtu, new.effective_mtu), - } - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -struct EffectiveIfConfig { - ipv4_addresses: BTreeSet, - ipv6_addresses: BTreeSet, - ipv4_routes: BTreeSet, - ipv6_routes: BTreeSet, - effective_mtu: Option, -} - -#[derive(Clone, Debug, Default)] -pub struct SharedIfConfig { - member_claims: BTreeMap, -} - -#[derive(Debug, thiserror::Error)] -#[error( - "shared virtual NIC destination {requested} overlaps member {member_id} destination {existing}" -)] -pub(crate) struct SharedIfConfigConflict { - requested: String, - member_id: SharedVirtualNicMemberId, - existing: String, -} - -impl SharedIfConfig { - pub(crate) fn ensure_disjoint( - &self, - member_id: SharedVirtualNicMemberId, - claims: &SharedIfConfigClaims, - ) -> anyhow::Result<()> { - let magic_dns = Ipv4Cidr::new(crate::instance::dns_server::MAGIC_DNS_FAKE_IP.parse()?, 32)?; - for (other_id, other) in &self.member_claims { - if *other_id == member_id { - continue; - } - for (requested, requested_is_route) in claims.ipv4_destinations() { - if let Some((existing, _)) = - other - .ipv4_destinations() - .find(|(existing, existing_is_route)| { - !(requested_is_route - && *existing_is_route - && requested == magic_dns - && *existing == magic_dns) - && ipv4_cidrs_overlap(requested, *existing) - }) - { - return Err(SharedIfConfigConflict { - requested: requested.to_string(), - member_id: *other_id, - existing: existing.to_string(), - } - .into()); - } - } - for requested in claims.ipv6_destinations() { - if let Some(existing) = other - .ipv6_destinations() - .find(|existing| ipv6_cidrs_overlap(requested, *existing)) - { - return Err(SharedIfConfigConflict { - requested: requested.to_string(), - member_id: *other_id, - existing: existing.to_string(), - } - .into()); - } - } - } - Ok(()) - } - - pub(crate) fn apply_member_claims( - &mut self, - member_id: SharedVirtualNicMemberId, - claims: SharedIfConfigClaims, - ) -> SharedIfConfigDelta { - let old = self.effective(); - self.member_claims.insert(member_id, claims); - SharedIfConfigDelta::between(&old, &self.effective()) - } - - pub fn remove_member( - &mut self, - member_id: SharedVirtualNicMemberId, - ) -> Option { - let old = self.effective(); - self.member_claims.remove(&member_id)?; - Some(SharedIfConfigDelta::between(&old, &self.effective())) - } - - pub fn effective_mtu(&self) -> Option { - self.member_claims - .values() - .filter_map(|claims| claims.mtu) - .min() - } - - pub fn owners_of_ipv4_route( - &self, - route: &SharedIpv4Route, - ) -> BTreeSet { - owners_for(&self.member_claims, |claims| &claims.ipv4_routes, route) - } - - pub fn owners_of_ipv4_address(&self, address: &Ipv4Inet) -> BTreeSet { - owners_for( - &self.member_claims, - |claims| &claims.ipv4_addresses, - address, - ) - } - - fn effective(&self) -> EffectiveIfConfig { - EffectiveIfConfig { - ipv4_addresses: self - .member_claims - .values() - .flat_map(|claims| claims.ipv4_addresses.iter().copied()) - .collect(), - ipv6_addresses: self - .member_claims - .values() - .flat_map(|claims| claims.ipv6_addresses.iter().copied()) - .collect(), - ipv4_routes: self - .member_claims - .values() - .flat_map(|claims| claims.ipv4_routes.iter().cloned()) - .collect(), - ipv6_routes: self - .member_claims - .values() - .flat_map(|claims| claims.ipv6_routes.iter().cloned()) - .collect(), - effective_mtu: self.effective_mtu(), - } - } - - fn claims_of(&self, member_id: SharedVirtualNicMemberId) -> SharedIfConfigClaims { - self.member_claims - .get(&member_id) - .cloned() - .unwrap_or_default() - } - - fn ipv4_route_source_hint(&self, route: &SharedIpv4Route) -> Option { - let route_inet = Ipv4Inet::new(route.address, route.prefix).ok(); - let mut fallback = None; - - for claims in self.member_claims.values() { - if !claims.ipv4_routes.contains(route) { - continue; - } - - for address in &claims.ipv4_addresses { - fallback.get_or_insert(address.address()); - if route_inet - .as_ref() - .is_some_and(|route_inet| route_inet.contains(&address.address())) - { - return Some(address.address()); - } - } - } - - fallback - } - - fn changed_ipv4_route_sources(&self, next: &Self) -> BTreeSet { - let old_routes = self.effective().ipv4_routes; - let next_routes = next.effective().ipv4_routes; - old_routes - .iter() - .filter(|route| { - next_routes.contains(*route) - && self.ipv4_route_source_hint(route) != next.ipv4_route_source_hint(route) - }) - .cloned() - .collect() - } -} - pub struct SharedVirtualNic { nic: Arc>, ifcfg: SharedIfConfig, @@ -360,14 +75,11 @@ impl SharedVirtualNic { self.valid.load(Ordering::Acquire) } - pub fn ifcfg(&self) -> &SharedIfConfig { + #[cfg(test)] + pub(crate) fn ifcfg(&self) -> &SharedIfConfig { &self.ifcfg } - pub fn ifcfg_mut(&mut self) -> &mut SharedIfConfig { - &mut self.ifcfg - } - pub fn nic(&self) -> Arc> { self.nic.clone() } @@ -429,10 +141,8 @@ impl SharedVirtualNic { if !self.is_current_member_registration(member_id, registration_id) { return Ok(()); } - #[cfg(mobile)] - return self.apply_member_claims_for_mobile(member_id, claims).await; - #[cfg(not(mobile))] - self.apply_member_claims(member_id, claims).await + self.apply_member_claims(member_id, claims, !cfg!(mobile)) + .await } async fn remove_member_registration_claims( @@ -459,45 +169,28 @@ impl SharedVirtualNic { &mut self, member_id: SharedVirtualNicMemberId, claims: SharedIfConfigClaims, + configure_os: bool, ) -> Result<(), Error> { self.ensure_valid()?; self.ifcfg.ensure_disjoint(member_id, &claims)?; let mut next_ifcfg = self.ifcfg.clone(); - let old_claims = self.ifcfg.claims_of(member_id); - let next_claims = claims.clone(); - let delta = next_ifcfg.apply_member_claims(member_id, claims); - self.sync_dispatcher_sources_for_ifcfg_update(member_id, &old_claims, &next_claims) - .await?; + next_ifcfg.apply_member_claims(member_id, claims.clone()); + if configure_os { + let old_claims = self.ifcfg.claims_of(member_id); + let active_claims = dispatcher_claims_for_ifcfg_transition(&old_claims, &claims); + self.sync_dispatcher_sources_for_claims(member_id, &active_claims) + .await?; - if let Err(err) = self.apply_ifcfg_delta(&delta, &next_ifcfg).await { - // Native operations may have already changed addresses or routes. - // Recreate the device rather than retaining inconsistent claims. - self.invalidate_and_shutdown_dispatcher().await; - return Err(err); + if let Err(err) = self.apply_ifcfg_update(&next_ifcfg).await { + // Native operations may have already changed addresses or routes. + // Recreate the device rather than retaining inconsistent claims. + self.invalidate_and_shutdown_dispatcher().await; + return Err(err); + } } - self.sync_dispatcher_sources_for_claims(member_id, &next_claims) - .await?; - self.ifcfg = next_ifcfg; - - Ok(()) - } - - #[cfg(any(mobile, test))] - async fn apply_member_claims_for_mobile( - &mut self, - member_id: SharedVirtualNicMemberId, - claims: SharedIfConfigClaims, - ) -> Result<(), Error> { - self.ensure_valid()?; - self.ifcfg.ensure_disjoint(member_id, &claims)?; - - let mut next_ifcfg = self.ifcfg.clone(); - let next_claims = claims.clone(); - next_ifcfg.apply_member_claims(member_id, claims); - - self.sync_dispatcher_sources_for_claims(member_id, &next_claims) + self.sync_dispatcher_sources_for_claims(member_id, &claims) .await?; self.ifcfg = next_ifcfg; @@ -510,29 +203,10 @@ impl SharedVirtualNic { registration_id: SharedVirtualNicMemberRegistrationId, mtu: u32, ) -> Result<(), Error> { - if !self.is_current_member_registration(member_id, registration_id) { - return Ok(()); - } - let mut claims = self.ifcfg.claims_of(member_id); claims.mtu = Some(mtu); - self.apply_member_claims(member_id, claims).await - } - - #[cfg(mobile)] - async fn apply_member_mtu_for_mobile_registration( - &mut self, - member_id: SharedVirtualNicMemberId, - registration_id: SharedVirtualNicMemberRegistrationId, - mtu: u32, - ) -> Result<(), Error> { - if !self.is_current_member_registration(member_id, registration_id) { - return Ok(()); - } - - let mut claims = self.ifcfg.claims_of(member_id); - claims.mtu = Some(mtu); - self.apply_member_claims_for_mobile(member_id, claims).await + self.apply_member_claims_for_registration(member_id, registration_id, claims) + .await } async fn remove_member_claims( @@ -542,13 +216,11 @@ impl SharedVirtualNic { self.ensure_valid()?; let mut next_ifcfg = self.ifcfg.clone(); - let Some(delta) = next_ifcfg.remove_member(member_id) else { + if !next_ifcfg.remove_member(member_id) { return Ok(()); - }; + } #[cfg(not(mobile))] - self.apply_ifcfg_delta(&delta, &next_ifcfg).await?; - #[cfg(mobile)] - drop(delta); + self.apply_ifcfg_update(&next_ifcfg).await?; if let Some(dispatcher) = &self.dispatcher { dispatcher.remove_sources(member_id).await?; } @@ -576,23 +248,25 @@ impl SharedVirtualNic { } } - async fn apply_ifcfg_delta( - &self, - delta: &SharedIfConfigDelta, - next_ifcfg: &SharedIfConfig, - ) -> Result<(), Error> { - let changed_routes = self.ifcfg.changed_ipv4_route_sources(next_ifcfg); - #[cfg(target_os = "linux")] + async fn apply_ifcfg_update(&self, next_ifcfg: &SharedIfConfig) -> Result<(), Error> { + let old_effective = self.ifcfg.effective(); let next_effective = next_ifcfg.effective(); + let delta = SharedIfConfigDelta::between(&old_effective, &next_effective); + let changed_routes: Vec<_> = old_effective + .ipv4_routes + .intersection(&next_effective.ipv4_routes) + .filter(|route| { + self.ifcfg.ipv4_route_source_hint(route) != next_ifcfg.ipv4_route_source_hint(route) + }) + .collect(); let nic = self.nic.lock().await; - for route in &delta.ipv4_routes.removed { - let source_hint = self.ifcfg.ipv4_route_source_hint(route); - ignore_removed_ifcfg_not_found( - remove_shared_ipv4_route(&nic, route, source_hint).await, - )?; - } - for route in &changed_routes { + for route in delta + .ipv4_routes + .removed + .iter() + .chain(changed_routes.iter().copied()) + { let source_hint = self.ifcfg.ipv4_route_source_hint(route); ignore_removed_ifcfg_not_found( remove_shared_ipv4_route(&nic, route, source_hint).await, @@ -617,10 +291,12 @@ impl SharedVirtualNic { nic.add_ipv6(ip.address(), ip.network_length() as i32) .await?; } - for route in &delta.ipv4_routes.added { - add_shared_ipv4_route(&nic, route, next_ifcfg).await?; - } - for route in &changed_routes { + for route in delta + .ipv4_routes + .added + .iter() + .chain(changed_routes.iter().copied()) + { add_shared_ipv4_route(&nic, route, next_ifcfg).await?; } for route in &delta.ipv6_routes.added { @@ -628,8 +304,8 @@ impl SharedVirtualNic { .await?; } - if let Some(mtu) = &delta.mtu { - nic.set_mtu(mtu.new.unwrap_or_else(|| nic.configured_mtu())) + if old_effective.mtu != next_effective.mtu { + nic.set_mtu(next_effective.mtu.unwrap_or_else(|| nic.configured_mtu())) .await?; } @@ -680,11 +356,7 @@ impl SharedVirtualNic { // AsyncDevice registers with the current I/O driver when it is created, // so moving only the dispatcher would still tie it to the first member. - let nic = self.nic.clone(); - let open = AbortOnDropHandle::new( - SHARED_VIRTUAL_NIC_RUNTIME.spawn(async move { nic.lock().await.create_dev().await }), - ); - let tunnel = open.await.map_err(anyhow::Error::from)??; + let tunnel = self.nic.lock().await.create_dev().await?; let dispatcher = SharedVirtualNicDispatcher::start( tunnel, self.member_tunnel_table.clone(), @@ -735,17 +407,6 @@ impl SharedVirtualNic { Ok(()) } - async fn sync_dispatcher_sources_for_ifcfg_update( - &self, - member_id: SharedVirtualNicMemberId, - old_claims: &SharedIfConfigClaims, - next_claims: &SharedIfConfigClaims, - ) -> Result<(), Error> { - let active_claims = dispatcher_claims_for_ifcfg_transition(old_claims, next_claims); - self.sync_dispatcher_sources_for_claims(member_id, &active_claims) - .await - } - async fn sync_dispatcher_sources_for_claims( &self, member_id: SharedVirtualNicMemberId, @@ -900,10 +561,6 @@ impl SharedVirtualNicMember { } } - pub fn member_id(&self) -> SharedVirtualNicMemberId { - self.member_id - } - #[cfg(mobile)] pub(crate) async fn claims(&self) -> SharedIfConfigClaims { self.shared_nic.lock().await.ifcfg.claims_of(self.member_id) @@ -941,15 +598,6 @@ impl SharedVirtualNicMember { self.shared_nic.clone() } - pub fn close_notifier(&self) -> Arc { - self.close_notifier.clone() - } - - #[cfg(test)] - fn configured_mtu_for_test(&self) -> u32 { - self.configured_mtu - } - pub async fn create_dev(&self) -> Result, Error> { let member = self.clone(); SHARED_VIRTUAL_NIC_RUNTIME @@ -996,7 +644,7 @@ impl SharedVirtualNicMember { .ensure_dispatcher_for_mobile(tun_fd, replace_tun_fd) .await?; shared_nic - .apply_member_mtu_for_mobile_registration( + .apply_member_mtu_for_registration( member.member_id, member.registration.registration_id, member.configured_mtu, @@ -1180,7 +828,8 @@ impl SharedVirtualNicRegistry { Self::default() } - pub fn get( + #[cfg(test)] + pub(crate) fn get( &self, dev_name: &str, config: &VirtualNicConfig, @@ -1206,14 +855,6 @@ impl SharedVirtualNicRegistry { Some(first) } - pub fn get_or_create( - &mut self, - dev_name: String, - config: VirtualNicConfig, - ) -> Arc> { - self.get_or_create_entry(dev_name, config).nic() - } - fn get_or_create_entry( &mut self, dev_name: String, @@ -1250,42 +891,6 @@ impl SharedVirtualNicRegistry { } } -fn item_delta(old: &BTreeSet, new: &BTreeSet) -> OwnedItemDelta -where - T: Ord + Clone, -{ - OwnedItemDelta { - added: new.difference(old).cloned().collect(), - removed: old.difference(new).cloned().collect(), - } -} - -fn owners_for( - claims: &BTreeMap, - select: fn(&SharedIfConfigClaims) -> &BTreeSet, - item: &T, -) -> BTreeSet -where - T: Ord, -{ - claims - .iter() - .filter_map(|(member_id, claims)| select(claims).contains(item).then_some(*member_id)) - .collect() -} - -fn mtu_delta(old: Option, new: Option) -> Option { - (old != new).then_some(SharedMtuChange { old, new }) -} - -fn ipv4_cidrs_overlap(left: Ipv4Cidr, right: Ipv4Cidr) -> bool { - left.contains(&right.first_address()) || right.contains(&left.first_address()) -} - -fn ipv6_cidrs_overlap(left: Ipv6Cidr, right: Ipv6Cidr) -> bool { - left.contains(&right.first_address()) || right.contains(&left.first_address()) -} - fn ipv4_inet(address: Ipv4Addr, prefix: i32) -> Result { let prefix = u8::try_from(prefix) .map_err(|_| anyhow::anyhow!("invalid IPv4 prefix length {}", prefix))?; @@ -1303,629 +908,4 @@ fn ipv6_inet(address: Ipv6Addr, prefix: i32) -> Result { } #[cfg(test)] -mod tests { - use std::str::FromStr as _; - - use crate::common::{ifcfg::IfConfiguerTrait, netns::NetNS}; - use tokio::sync::Notify; - - use super::*; - - struct FailingRemoveIpIfConfiger; - - #[async_trait::async_trait] - impl IfConfiguerTrait for FailingRemoveIpIfConfiger { - async fn remove_ip(&self, _name: &str, _ip: Option) -> Result<(), Error> { - Err(anyhow::anyhow!("forced remove_ip failure").into()) - } - } - - fn member_id(n: u128) -> SharedVirtualNicMemberId { - uuid::Uuid::from_u128(n) - } - - fn claims_with_ipv4_route(route: SharedIpv4Route, mtu: Option) -> SharedIfConfigClaims { - SharedIfConfigClaims { - ipv4_routes: BTreeSet::from([route]), - mtu, - ..Default::default() - } - } - - fn claims_with_ipv4_address_and_route( - address: Ipv4Inet, - route: SharedIpv4Route, - ) -> SharedIfConfigClaims { - SharedIfConfigClaims { - ipv4_addresses: BTreeSet::from([address]), - ipv4_routes: BTreeSet::from([route]), - ..Default::default() - } - } - - fn virtual_nic_config() -> VirtualNicConfig { - VirtualNicConfig::new(String::new(), 1500, NetNS::new(None)) - } - - fn virtual_nic_config_with_mtu(mtu: u32) -> VirtualNicConfig { - VirtualNicConfig::new(String::new(), mtu, NetNS::new(None)) - } - - fn virtual_nic_config_in_netns(net_ns: &str) -> VirtualNicConfig { - VirtualNicConfig::new(String::new(), 1500, NetNS::new(Some(net_ns.to_owned()))) - } - - #[test] - fn duplicate_routes_keep_owner_sets_and_single_os_delta() { - let route = SharedIpv4Route::new(Ipv4Addr::new(100, 100, 100, 101), 32, None); - let first = member_id(1); - let second = member_id(2); - let mut ifcfg = SharedIfConfig::default(); - - let first_delta = - ifcfg.apply_member_claims(first, claims_with_ipv4_route(route.clone(), Some(1400))); - let second_delta = - ifcfg.apply_member_claims(second, claims_with_ipv4_route(route.clone(), Some(1300))); - - assert_eq!( - first_delta.ipv4_routes.added, - BTreeSet::from([route.clone()]) - ); - assert!(second_delta.ipv4_routes.added.is_empty()); - assert_eq!( - ifcfg.owners_of_ipv4_route(&route), - BTreeSet::from([first, second]) - ); - assert_eq!(ifcfg.effective_mtu(), Some(1300)); - } - - #[test] - fn removing_one_owner_keeps_shared_route_until_last_owner_leaves() { - let route = SharedIpv4Route::new(Ipv4Addr::new(100, 100, 100, 101), 32, None); - let first = member_id(1); - let second = member_id(2); - let mut ifcfg = SharedIfConfig::default(); - ifcfg.apply_member_claims(first, claims_with_ipv4_route(route.clone(), None)); - ifcfg.apply_member_claims(second, claims_with_ipv4_route(route.clone(), None)); - - let first_delta = ifcfg.remove_member(first).unwrap(); - let second_delta = ifcfg.remove_member(second).unwrap(); - - assert!(first_delta.ipv4_routes.removed.is_empty()); - assert_eq!( - second_delta.ipv4_routes.removed, - BTreeSet::from([route.clone()]) - ); - assert!(ifcfg.owners_of_ipv4_route(&route).is_empty()); - } - - #[tokio::test] - async fn overlapping_member_claims_leave_existing_configuration_intact() { - let first = member_id(1); - let second = member_id(2); - let mut shared_nic = SharedVirtualNic::new(virtual_nic_config()); - shared_nic.ifcfg_mut().apply_member_claims( - first, - SharedIfConfigClaims { - ipv4_routes: BTreeSet::from([SharedIpv4Route::new( - Ipv4Addr::new(10, 144, 0, 0), - 16, - None, - )]), - ..Default::default() - }, - ); - let original = shared_nic.ifcfg().effective(); - - let err = shared_nic - .apply_member_claims( - second, - SharedIfConfigClaims { - ipv4_addresses: BTreeSet::from(["10.144.1.2/24".parse().unwrap()]), - ..Default::default() - }, - ) - .await - .unwrap_err(); - - assert!(err.to_string().contains("overlaps member")); - assert_eq!(shared_nic.ifcfg().effective(), original); - } - - #[test] - fn shared_magic_dns_route_is_not_a_member_conflict() { - let first = member_id(1); - let second = member_id(2); - let route = SharedIpv4Route::new(Ipv4Addr::new(100, 100, 100, 101), 32, None); - let claims = claims_with_ipv4_route(route, None); - let mut ifcfg = SharedIfConfig::default(); - ifcfg.apply_member_claims(first, claims.clone()); - - assert!(ifcfg.ensure_disjoint(second, &claims).is_ok()); - } - - #[test] - fn magic_dns_address_conflicts_with_other_members() { - let address_claims = SharedIfConfigClaims { - ipv4_addresses: BTreeSet::from(["100.100.100.101/32".parse().unwrap()]), - ..Default::default() - }; - let route_claims = claims_with_ipv4_route( - SharedIpv4Route::new(Ipv4Addr::new(100, 100, 100, 101), 32, None), - None, - ); - - for (first, second) in [ - (&address_claims, &address_claims), - (&address_claims, &route_claims), - (&route_claims, &address_claims), - ] { - let mut ifcfg = SharedIfConfig::default(); - ifcfg.apply_member_claims(member_id(1), first.clone()); - - assert!(ifcfg.ensure_disjoint(member_id(2), second).is_err()); - } - } - - #[tokio::test] - async fn mobile_claims_do_not_configure_the_os_interface() { - // There is no OS interface or ifname; native ifcfg calls would fail. - let mut shared_nic = SharedVirtualNic::new(virtual_nic_config()); - let member = member_id(1); - let claims = SharedIfConfigClaims { - ipv4_addresses: BTreeSet::from(["10.144.1.1/24".parse().unwrap()]), - ipv6_addresses: BTreeSet::from(["fd00:1::1/64".parse().unwrap()]), - ipv4_routes: BTreeSet::from([SharedIpv4Route::new( - Ipv4Addr::new(10, 145, 0, 0), - 16, - None, - )]), - ipv6_routes: BTreeSet::from([SharedIpv6Route::new( - "fd00:2::".parse().unwrap(), - 64, - None, - )]), - mtu: Some(1300), - }; - - #[cfg(mobile)] - { - let registration = uuid::Uuid::new_v4(); - shared_nic.member_registrations.insert(member, registration); - shared_nic.link_up().await.unwrap(); - shared_nic - .apply_member_claims_for_registration(member, registration, claims.clone()) - .await - .unwrap(); - } - #[cfg(not(mobile))] - shared_nic - .apply_member_claims_for_mobile(member, claims.clone()) - .await - .unwrap(); - assert_eq!(shared_nic.ifcfg().claims_of(member), claims); - assert!(shared_nic.is_valid()); - - shared_nic - .apply_member_claims_for_mobile(member, SharedIfConfigClaims::default()) - .await - .unwrap(); - assert_eq!( - shared_nic.ifcfg().claims_of(member), - SharedIfConfigClaims::default() - ); - } - - #[tokio::test] - async fn conflicting_mobile_member_leaves_no_partial_claims() { - let mut shared_nic = SharedVirtualNic::new(virtual_nic_config()); - let healthy = SharedIfConfigClaims { - ipv4_addresses: BTreeSet::from(["10.144.1.1/24".parse().unwrap()]), - ipv6_addresses: BTreeSet::from(["fd00:1::1/64".parse().unwrap()]), - ..Default::default() - }; - shared_nic - .apply_member_claims_for_mobile(member_id(1), healthy.clone()) - .await - .unwrap(); - let conflicting = SharedIfConfigClaims { - // IPv4 is disjoint, but the later IPv6 claim overlaps the healthy member. - ipv4_addresses: BTreeSet::from(["10.144.2.1/24".parse().unwrap()]), - ipv6_addresses: BTreeSet::from(["fd00:1::2/64".parse().unwrap()]), - ..Default::default() - }; - assert!( - shared_nic - .apply_member_claims_for_mobile(member_id(2), conflicting) - .await - .is_err() - ); - assert_eq!(shared_nic.ifcfg().claims_of(member_id(1)), healthy); - assert_eq!( - shared_nic.ifcfg().claims_of(member_id(2)), - SharedIfConfigClaims::default() - ); - assert!(shared_nic.is_valid()); - } - - #[test] - fn member_claim_update_tracks_ip_ownership() { - let first_ip = Ipv4Inet::from_str("10.30.0.2/24").unwrap(); - let second_ip = Ipv4Inet::from_str("10.30.0.3/24").unwrap(); - let member = member_id(1); - let mut ifcfg = SharedIfConfig::default(); - - let first_delta = ifcfg.apply_member_claims( - member, - SharedIfConfigClaims { - ipv4_addresses: BTreeSet::from([first_ip]), - ..Default::default() - }, - ); - let second_delta = ifcfg.apply_member_claims( - member, - SharedIfConfigClaims { - ipv4_addresses: BTreeSet::from([second_ip]), - ..Default::default() - }, - ); - - assert_eq!(first_delta.ipv4_addresses.added, BTreeSet::from([first_ip])); - assert_eq!( - second_delta.ipv4_addresses.removed, - BTreeSet::from([first_ip]) - ); - assert_eq!( - second_delta.ipv4_addresses.added, - BTreeSet::from([second_ip]) - ); - assert_eq!( - ifcfg.owners_of_ipv4_address(&second_ip), - BTreeSet::from([member]) - ); - } - - #[test] - fn ipv4_route_source_hint_prefers_address_inside_route() { - let route = SharedIpv4Route::new(Ipv4Addr::new(10, 90, 1, 0), 24, None); - let member = member_id(1); - let mut ifcfg = SharedIfConfig::default(); - - ifcfg.apply_member_claims( - member, - SharedIfConfigClaims { - ipv4_addresses: BTreeSet::from([ - Ipv4Inet::from_str("10.1.1.1/24").unwrap(), - Ipv4Inet::from_str("10.90.1.1/24").unwrap(), - ]), - ipv4_routes: BTreeSet::from([route.clone()]), - ..Default::default() - }, - ); - - assert_eq!( - ifcfg.ipv4_route_source_hint(&route), - Some(Ipv4Addr::new(10, 90, 1, 1)) - ); - } - - #[test] - fn adding_better_ipv4_route_owner_marks_source_change() { - let route = SharedIpv4Route::new(Ipv4Addr::new(10, 90, 2, 0), 24, None); - let first = member_id(1); - let second = member_id(2); - let mut ifcfg = SharedIfConfig::default(); - ifcfg.apply_member_claims( - first, - claims_with_ipv4_address_and_route( - Ipv4Inet::from_str("10.1.2.1/24").unwrap(), - route.clone(), - ), - ); - let old = ifcfg.clone(); - - let delta = ifcfg.apply_member_claims( - second, - claims_with_ipv4_address_and_route( - Ipv4Inet::from_str("10.90.2.1/24").unwrap(), - route.clone(), - ), - ); - - assert!(delta.ipv4_routes.added.is_empty()); - assert_eq!( - old.changed_ipv4_route_sources(&ifcfg), - BTreeSet::from([route.clone()]) - ); - assert_eq!( - old.ipv4_route_source_hint(&route), - Some(Ipv4Addr::new(10, 1, 2, 1)) - ); - assert_eq!( - ifcfg.ipv4_route_source_hint(&route), - Some(Ipv4Addr::new(10, 90, 2, 1)) - ); - } - - #[test] - fn removing_ipv4_route_owner_marks_source_change_when_route_remains() { - let route = SharedIpv4Route::new(Ipv4Addr::new(10, 90, 3, 0), 24, None); - let first = member_id(1); - let second = member_id(2); - let mut ifcfg = SharedIfConfig::default(); - ifcfg.apply_member_claims( - first, - claims_with_ipv4_address_and_route( - Ipv4Inet::from_str("10.1.3.1/24").unwrap(), - route.clone(), - ), - ); - ifcfg.apply_member_claims( - second, - claims_with_ipv4_address_and_route( - Ipv4Inet::from_str("10.90.3.1/24").unwrap(), - route.clone(), - ), - ); - let old = ifcfg.clone(); - - let delta = ifcfg.remove_member(second).unwrap(); - - assert!(delta.ipv4_routes.removed.is_empty()); - assert_eq!( - old.changed_ipv4_route_sources(&ifcfg), - BTreeSet::from([route.clone()]) - ); - assert_eq!( - old.ipv4_route_source_hint(&route), - Some(Ipv4Addr::new(10, 90, 3, 1)) - ); - assert_eq!( - ifcfg.ipv4_route_source_hint(&route), - Some(Ipv4Addr::new(10, 1, 3, 1)) - ); - } - - #[test] - fn removing_ipv4_route_keeps_old_source_hint_available() { - let kept_route = SharedIpv4Route::new(Ipv4Addr::new(10, 90, 4, 0), 24, Some(10)); - let removed_route = SharedIpv4Route::new(Ipv4Addr::new(10, 90, 4, 0), 24, Some(20)); - let member = member_id(1); - let address = Ipv4Inet::from_str("10.90.4.1/24").unwrap(); - let mut ifcfg = SharedIfConfig::default(); - ifcfg.apply_member_claims( - member, - SharedIfConfigClaims { - ipv4_addresses: BTreeSet::from([address]), - ipv4_routes: BTreeSet::from([kept_route.clone(), removed_route.clone()]), - ..Default::default() - }, - ); - let old = ifcfg.clone(); - - let delta = ifcfg.apply_member_claims( - member, - SharedIfConfigClaims { - ipv4_addresses: BTreeSet::from([address]), - ipv4_routes: BTreeSet::from([kept_route.clone()]), - ..Default::default() - }, - ); - - assert_eq!( - delta.ipv4_routes.removed, - BTreeSet::from([removed_route.clone()]) - ); - assert_eq!( - old.ipv4_route_source_hint(&removed_route), - Some(Ipv4Addr::new(10, 90, 4, 1)) - ); - } - - #[test] - fn shared_virtual_nic_wraps_virtual_nic_and_tracks_ifcfg() { - let mut shared_nic = SharedVirtualNic::new(virtual_nic_config()); - let member = member_id(1); - let route = SharedIpv4Route::new(Ipv4Addr::new(10, 40, 0, 0), 24, None); - - shared_nic - .ifcfg_mut() - .apply_member_claims(member, claims_with_ipv4_route(route.clone(), None)); - - assert_eq!( - shared_nic.ifcfg().owners_of_ipv4_route(&route), - BTreeSet::from([member]) - ); - drop(shared_nic.nic()); - } - - #[tokio::test] - async fn stale_member_registration_cleanup_keeps_current_claims() { - let mut shared_nic = SharedVirtualNic::new(virtual_nic_config()); - let member = member_id(1); - let old_registration = uuid::Uuid::from_u128(10); - let current_registration = uuid::Uuid::from_u128(11); - let ip = Ipv4Inet::from_str("10.50.0.2/24").unwrap(); - - shared_nic - .member_registrations - .insert(member, current_registration); - shared_nic.ifcfg_mut().apply_member_claims( - member, - SharedIfConfigClaims { - ipv4_addresses: BTreeSet::from([ip]), - ..Default::default() - }, - ); - - shared_nic - .remove_member_registration_claims(member, old_registration) - .await - .unwrap(); - - assert_eq!( - shared_nic.ifcfg().owners_of_ipv4_address(&ip), - BTreeSet::from([member]) - ); - assert_eq!( - shared_nic.member_registrations.get(&member), - Some(¤t_registration) - ); - } - - #[tokio::test] - async fn failed_member_registration_cleanup_invalidates_shared_nic() { - let mut shared_nic = SharedVirtualNic::new(virtual_nic_config()); - let member = member_id(1); - let registration = uuid::Uuid::from_u128(10); - let ip = Ipv4Inet::from_str("10.60.0.2/24").unwrap(); - - shared_nic.member_registrations.insert(member, registration); - shared_nic.ifcfg_mut().apply_member_claims( - member, - SharedIfConfigClaims { - ipv4_addresses: BTreeSet::from([ip]), - ..Default::default() - }, - ); - let nic = shared_nic.nic(); - let mut nic = nic.lock().await; - nic.set_ifname_for_test("et0".to_string()); - nic.set_ifcfg_for_test(Box::new(FailingRemoveIpIfConfiger)); - drop(nic); - - let result = shared_nic - .remove_member_registration_claims(member, registration) - .await; - - assert!(result.is_err()); - assert!(!shared_nic.is_valid()); - assert!(!shared_nic.member_registrations.contains_key(&member)); - assert_eq!( - shared_nic.ifcfg().owners_of_ipv4_address(&ip), - BTreeSet::from([member]) - ); - } - - #[tokio::test] - async fn failed_registration_replacement_keeps_old_registration_and_invalidates() { - let mut shared_nic = SharedVirtualNic::new(virtual_nic_config()); - let member = member_id(1); - let old_registration = uuid::Uuid::from_u128(10); - let next_registration = uuid::Uuid::from_u128(11); - let ip = Ipv4Inet::from_str("10.70.0.2/24").unwrap(); - - shared_nic - .member_registrations - .insert(member, old_registration); - shared_nic.ifcfg_mut().apply_member_claims( - member, - SharedIfConfigClaims { - ipv4_addresses: BTreeSet::from([ip]), - ..Default::default() - }, - ); - let nic = shared_nic.nic(); - let mut nic = nic.lock().await; - nic.set_ifname_for_test("et0".to_string()); - nic.set_ifcfg_for_test(Box::new(FailingRemoveIpIfConfiger)); - drop(nic); - - let result = shared_nic - .attach_member_registration(member, next_registration) - .await; - - assert!(result.is_err()); - assert!(!shared_nic.is_valid()); - assert_eq!( - shared_nic.member_registrations.get(&member), - Some(&old_registration) - ); - } - - #[test] - fn registry_reuses_shared_virtual_nic_for_same_dev_name_and_netns() { - let mut registry = SharedVirtualNicRegistry::new(); - - let first = registry.get_or_create("et0".to_string(), virtual_nic_config()); - let second = registry.get_or_create("et0".to_string(), virtual_nic_config()); - - assert!(Arc::ptr_eq(&first, &second)); - } - - #[test] - fn registry_keeps_same_dev_name_in_different_netns_separate() { - let mut registry = SharedVirtualNicRegistry::new(); - - let first = registry.get_or_create("et0".to_string(), virtual_nic_config_in_netns("net-a")); - let second = - registry.get_or_create("et0".to_string(), virtual_nic_config_in_netns("net-b")); - - assert!(!Arc::ptr_eq(&first, &second)); - } - - #[test] - fn registry_keeps_different_dev_names_separate() { - let mut registry = SharedVirtualNicRegistry::new(); - - let first = registry.get_or_create("et0".to_string(), virtual_nic_config()); - let second = registry.get_or_create("et1".to_string(), virtual_nic_config()); - - assert!(!Arc::ptr_eq(&first, &second)); - } - - #[test] - fn registry_replaces_invalid_shared_virtual_nic() { - let mut registry = SharedVirtualNicRegistry::new(); - - let first = registry.get_or_create("et0".to_string(), virtual_nic_config()); - first.try_lock().unwrap().mark_invalid(); - let second = registry.get_or_create("et0".to_string(), virtual_nic_config()); - - assert!(!Arc::ptr_eq(&first, &second)); - assert!( - registry - .get("et0", &virtual_nic_config()) - .is_some_and(|nic| Arc::ptr_eq(&nic, &second)) - ); - } - - #[test] - fn registry_create_member_uses_registered_shared_virtual_nic() { - let mut registry = SharedVirtualNicRegistry::new(); - let member_id = member_id(1); - - let member = registry.create_member( - "et0".to_string(), - virtual_nic_config(), - member_id, - Arc::new(Notify::new()), - ); - let shared_nic = registry.get("et0", &virtual_nic_config()).unwrap(); - - assert_eq!(member.member_id(), member_id); - assert!(Arc::ptr_eq(&member.shared_nic(), &shared_nic)); - } - - #[test] - fn registry_create_member_keeps_member_configured_mtu() { - let mut registry = SharedVirtualNicRegistry::new(); - - let first = registry.create_member( - "et0".to_string(), - virtual_nic_config_with_mtu(1400), - member_id(1), - Arc::new(Notify::new()), - ); - let second = registry.create_member( - "et0".to_string(), - virtual_nic_config_with_mtu(1300), - member_id(2), - Arc::new(Notify::new()), - ); - - assert_eq!(first.configured_mtu_for_test(), 1400); - assert_eq!(second.configured_mtu_for_test(), 1300); - assert!(Arc::ptr_eq(&first.shared_nic(), &second.shared_nic())); - } -} +mod tests; diff --git a/easytier/src/instance/shared_virtual_nic/ifcfg.rs b/easytier/src/instance/shared_virtual_nic/ifcfg.rs new file mode 100644 index 00000000..7b57da29 --- /dev/null +++ b/easytier/src/instance/shared_virtual_nic/ifcfg.rs @@ -0,0 +1,272 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + net::{Ipv4Addr, Ipv6Addr}, +}; + +use cidr::{Ipv4Cidr, Ipv4Inet, Ipv6Cidr, Ipv6Inet}; + +use super::SharedVirtualNicMemberId; + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct SharedIpv4Route { + pub address: Ipv4Addr, + pub prefix: u8, + pub cost: Option, +} + +impl SharedIpv4Route { + pub fn new(address: Ipv4Addr, prefix: u8, cost: Option) -> Self { + Self { + address, + prefix, + cost, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct SharedIpv6Route { + pub address: Ipv6Addr, + pub prefix: u8, + pub cost: Option, +} + +impl SharedIpv6Route { + pub fn new(address: Ipv6Addr, prefix: u8, cost: Option) -> Self { + Self { + address, + prefix, + cost, + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SharedIfConfigClaims { + pub ipv4_addresses: BTreeSet, + pub ipv6_addresses: BTreeSet, + pub ipv4_routes: BTreeSet, + pub ipv6_routes: BTreeSet, + pub mtu: Option, +} + +impl SharedIfConfigClaims { + fn ipv4_destinations(&self) -> impl Iterator + '_ { + self.ipv4_addresses + .iter() + .map(|ip| (ip.network(), false)) + .chain( + self.ipv4_routes + .iter() + .filter_map(|route| Ipv4Inet::new(route.address, route.prefix).ok()) + .map(|route| (route.network(), true)), + ) + } + + fn ipv6_destinations(&self) -> impl Iterator + '_ { + self.ipv6_addresses.iter().map(Ipv6Inet::network).chain( + self.ipv6_routes + .iter() + .filter_map(|route| Ipv6Inet::new(route.address, route.prefix).ok()) + .map(|route| route.network()), + ) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct OwnedItemDelta { + pub added: BTreeSet, + pub removed: BTreeSet, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct SharedIfConfigDelta { + pub ipv4_addresses: OwnedItemDelta, + pub ipv6_addresses: OwnedItemDelta, + pub ipv4_routes: OwnedItemDelta, + pub ipv6_routes: OwnedItemDelta, +} + +impl SharedIfConfigDelta { + pub(super) fn between(old: &SharedIfConfigClaims, new: &SharedIfConfigClaims) -> Self { + Self { + ipv4_addresses: item_delta(&old.ipv4_addresses, &new.ipv4_addresses), + ipv6_addresses: item_delta(&old.ipv6_addresses, &new.ipv6_addresses), + ipv4_routes: item_delta(&old.ipv4_routes, &new.ipv4_routes), + ipv6_routes: item_delta(&old.ipv6_routes, &new.ipv6_routes), + } + } +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct SharedIfConfig { + pub(super) member_claims: BTreeMap, +} + +#[derive(Debug, thiserror::Error)] +#[error( + "shared virtual NIC destination {requested} overlaps member {member_id} destination {existing}" +)] +pub(crate) struct SharedIfConfigConflict { + requested: String, + member_id: SharedVirtualNicMemberId, + existing: String, +} + +impl SharedIfConfig { + pub(crate) fn ensure_disjoint( + &self, + member_id: SharedVirtualNicMemberId, + claims: &SharedIfConfigClaims, + ) -> anyhow::Result<()> { + let magic_dns = Ipv4Cidr::new(crate::instance::dns_server::MAGIC_DNS_FAKE_IP.parse()?, 32)?; + for (other_id, other) in &self.member_claims { + if *other_id == member_id { + continue; + } + for (requested, requested_is_route) in claims.ipv4_destinations() { + if let Some((existing, _)) = + other + .ipv4_destinations() + .find(|(existing, existing_is_route)| { + !(requested_is_route + && *existing_is_route + && requested == magic_dns + && *existing == magic_dns) + && ipv4_cidrs_overlap(requested, *existing) + }) + { + return Err(SharedIfConfigConflict { + requested: requested.to_string(), + member_id: *other_id, + existing: existing.to_string(), + } + .into()); + } + } + for requested in claims.ipv6_destinations() { + if let Some(existing) = other + .ipv6_destinations() + .find(|existing| ipv6_cidrs_overlap(requested, *existing)) + { + return Err(SharedIfConfigConflict { + requested: requested.to_string(), + member_id: *other_id, + existing: existing.to_string(), + } + .into()); + } + } + } + Ok(()) + } + + pub(crate) fn apply_member_claims( + &mut self, + member_id: SharedVirtualNicMemberId, + claims: SharedIfConfigClaims, + ) { + self.member_claims.insert(member_id, claims); + } + + pub(super) fn remove_member(&mut self, member_id: SharedVirtualNicMemberId) -> bool { + self.member_claims.remove(&member_id).is_some() + } + + fn effective_mtu(&self) -> Option { + self.member_claims + .values() + .filter_map(|claims| claims.mtu) + .min() + } + + #[cfg(test)] + pub(crate) fn owners_of_ipv4_route( + &self, + route: &SharedIpv4Route, + ) -> BTreeSet { + self.member_claims + .iter() + .filter_map(|(member_id, claims)| { + claims.ipv4_routes.contains(route).then_some(*member_id) + }) + .collect() + } + + pub(super) fn effective(&self) -> SharedIfConfigClaims { + SharedIfConfigClaims { + ipv4_addresses: self + .member_claims + .values() + .flat_map(|claims| claims.ipv4_addresses.iter().copied()) + .collect(), + ipv6_addresses: self + .member_claims + .values() + .flat_map(|claims| claims.ipv6_addresses.iter().copied()) + .collect(), + ipv4_routes: self + .member_claims + .values() + .flat_map(|claims| claims.ipv4_routes.iter().cloned()) + .collect(), + ipv6_routes: self + .member_claims + .values() + .flat_map(|claims| claims.ipv6_routes.iter().cloned()) + .collect(), + mtu: self.effective_mtu(), + } + } + + pub(super) fn claims_of(&self, member_id: SharedVirtualNicMemberId) -> SharedIfConfigClaims { + self.member_claims + .get(&member_id) + .cloned() + .unwrap_or_default() + } + + pub(super) fn ipv4_route_source_hint(&self, route: &SharedIpv4Route) -> Option { + let route_inet = Ipv4Inet::new(route.address, route.prefix).ok(); + let mut fallback = None; + + for claims in self.member_claims.values() { + if !claims.ipv4_routes.contains(route) { + continue; + } + + for address in &claims.ipv4_addresses { + fallback.get_or_insert(address.address()); + if route_inet + .as_ref() + .is_some_and(|route_inet| route_inet.contains(&address.address())) + { + return Some(address.address()); + } + } + } + + fallback + } +} + +fn item_delta(old: &BTreeSet, new: &BTreeSet) -> OwnedItemDelta +where + T: Ord + Clone, +{ + OwnedItemDelta { + added: new.difference(old).cloned().collect(), + removed: old.difference(new).cloned().collect(), + } +} + +fn ipv4_cidrs_overlap(left: Ipv4Cidr, right: Ipv4Cidr) -> bool { + left.contains(&right.first_address()) || right.contains(&left.first_address()) +} + +fn ipv6_cidrs_overlap(left: Ipv6Cidr, right: Ipv6Cidr) -> bool { + left.contains(&right.first_address()) || right.contains(&left.first_address()) +} + +#[cfg(test)] +mod tests; diff --git a/easytier/src/instance/shared_virtual_nic/ifcfg/tests.rs b/easytier/src/instance/shared_virtual_nic/ifcfg/tests.rs new file mode 100644 index 00000000..1d38c5b5 --- /dev/null +++ b/easytier/src/instance/shared_virtual_nic/ifcfg/tests.rs @@ -0,0 +1,128 @@ +use super::*; + +fn member_id(n: u128) -> SharedVirtualNicMemberId { + uuid::Uuid::from_u128(n) +} + +fn route(cidr: &str) -> SharedIpv4Route { + let ip: Ipv4Inet = cidr.parse().unwrap(); + SharedIpv4Route::new(ip.address(), ip.network_length(), None) +} + +#[test] +fn shared_route_and_mtu_follow_member_ownership() { + let route = route("100.100.100.101/32"); + let mut ifcfg = SharedIfConfig::default(); + for (member, mtu) in [(member_id(1), 1400), (member_id(2), 1300)] { + let old = ifcfg.effective(); + ifcfg.apply_member_claims( + member, + SharedIfConfigClaims { + ipv4_routes: BTreeSet::from([route.clone()]), + mtu: Some(mtu), + ..Default::default() + }, + ); + let next = ifcfg.effective(); + let delta = SharedIfConfigDelta::between(&old, &next); + assert_eq!( + delta.ipv4_routes.added.contains(&route), + member == member_id(1) + ); + assert!(delta.ipv4_routes.removed.is_empty()); + assert_eq!(next.mtu, Some(mtu)); + } + assert_eq!( + ifcfg.owners_of_ipv4_route(&route), + BTreeSet::from([member_id(1), member_id(2)]) + ); + + for (member, expected_mtu) in [(member_id(2), Some(1400)), (member_id(1), None)] { + let old = ifcfg.effective(); + assert!(ifcfg.remove_member(member)); + let next = ifcfg.effective(); + let delta = SharedIfConfigDelta::between(&old, &next); + assert!(delta.ipv4_routes.added.is_empty()); + assert_eq!( + delta.ipv4_routes.removed.contains(&route), + expected_mtu.is_none() + ); + assert_eq!(next.mtu, expected_mtu); + } + assert!(ifcfg.owners_of_ipv4_route(&route).is_empty()); + assert!(!ifcfg.remove_member(member_id(1))); +} + +#[test] +fn replacing_member_claims_releases_previous_addresses() { + let first_ip: Ipv4Inet = "10.30.0.2/24".parse().unwrap(); + let second_ip: Ipv4Inet = "10.30.0.3/24".parse().unwrap(); + let mut ifcfg = SharedIfConfig::default(); + for ip in [first_ip, second_ip] { + let old = ifcfg.effective(); + ifcfg.apply_member_claims( + member_id(1), + SharedIfConfigClaims { + ipv4_addresses: BTreeSet::from([ip]), + ..Default::default() + }, + ); + let delta = SharedIfConfigDelta::between(&old, &ifcfg.effective()); + assert_eq!(delta.ipv4_addresses.added, BTreeSet::from([ip])); + assert_eq!(delta.ipv4_addresses.removed, old.ipv4_addresses); + assert_eq!( + ifcfg.claims_of(member_id(1)).ipv4_addresses, + BTreeSet::from([ip]) + ); + } +} + +#[test] +fn only_magic_dns_routes_can_overlap_between_members() { + let address_claims = SharedIfConfigClaims { + ipv4_addresses: BTreeSet::from(["100.100.100.101/32".parse().unwrap()]), + ..Default::default() + }; + let route_claims = SharedIfConfigClaims { + ipv4_routes: BTreeSet::from([route("100.100.100.101/32")]), + ..Default::default() + }; + + for (first, second, allowed) in [ + (&route_claims, &route_claims, true), + (&address_claims, &address_claims, false), + (&address_claims, &route_claims, false), + (&route_claims, &address_claims, false), + ] { + let mut ifcfg = SharedIfConfig::default(); + ifcfg.apply_member_claims(member_id(1), first.clone()); + assert_eq!(ifcfg.ensure_disjoint(member_id(2), second).is_ok(), allowed); + assert!(ifcfg.ensure_disjoint(member_id(1), second).is_ok()); + } +} + +#[test] +fn route_source_hint_prefers_an_owner_address_inside_the_route() { + let route = route("10.90.1.0/24"); + let mut ifcfg = SharedIfConfig::default(); + for (addresses, expected) in [ + (vec!["10.1.1.1/24"], "10.1.1.1"), + (vec!["10.1.1.1/24", "10.90.1.1/24"], "10.90.1.1"), + ] { + ifcfg.apply_member_claims( + member_id(1), + SharedIfConfigClaims { + ipv4_addresses: addresses + .into_iter() + .map(|ip| ip.parse().unwrap()) + .collect(), + ipv4_routes: BTreeSet::from([route.clone()]), + ..Default::default() + }, + ); + assert_eq!( + ifcfg.ipv4_route_source_hint(&route), + Some(expected.parse().unwrap()) + ); + } +} diff --git a/easytier/src/instance/shared_virtual_nic/tests.rs b/easytier/src/instance/shared_virtual_nic/tests.rs new file mode 100644 index 00000000..03aa470a --- /dev/null +++ b/easytier/src/instance/shared_virtual_nic/tests.rs @@ -0,0 +1,322 @@ +use std::sync::Mutex as StdMutex; + +use crate::common::{ifcfg::IfConfiguerTrait, netns::NetNS}; + +use super::*; + +const MEMBER: SharedVirtualNicMemberId = uuid::Uuid::from_u128(1); +const REGISTRATION: SharedVirtualNicMemberRegistrationId = uuid::Uuid::from_u128(10); + +fn virtual_nic_config(mtu: u32, netns: Option<&str>) -> VirtualNicConfig { + VirtualNicConfig::new("et0".to_owned(), mtu, NetNS::new(netns.map(str::to_owned))) +} + +fn address_claims(ip: &str) -> SharedIfConfigClaims { + SharedIfConfigClaims { + ipv4_addresses: BTreeSet::from([ip.parse().unwrap()]), + ..Default::default() + } +} + +fn registered_nic() -> SharedVirtualNic { + let mut nic = SharedVirtualNic::new(virtual_nic_config(1500, None)); + nic.member_registrations.insert(MEMBER, REGISTRATION); + nic.ifcfg + .apply_member_claims(MEMBER, address_claims("10.50.0.2/24")); + nic +} + +#[derive(Debug, PartialEq, Eq)] +enum RouteChange { + Add(SharedIpv4Route, Option), + Remove(SharedIpv4Route, Option), +} + +#[derive(Clone, Default)] +struct RecordingIfConfiger { + routes: Arc>>, + fail_remove_ip: bool, +} + +#[async_trait::async_trait] +impl IfConfiguerTrait for RecordingIfConfiger { + async fn add_ipv4_route_with_source_hint( + &self, + _name: &str, + address: Ipv4Addr, + prefix: u8, + cost: Option, + source_hint: Option, + ) -> Result<(), Error> { + self.routes.lock().unwrap().push(RouteChange::Add( + SharedIpv4Route::new(address, prefix, cost), + source_hint, + )); + Ok(()) + } + + async fn remove_ipv4_route_with_cost_and_source_hint( + &self, + _name: &str, + address: Ipv4Addr, + prefix: u8, + cost: Option, + source_hint: Option, + ) -> Result<(), Error> { + self.routes.lock().unwrap().push(RouteChange::Remove( + SharedIpv4Route::new(address, prefix, cost), + source_hint, + )); + Ok(()) + } + + async fn remove_ip(&self, _name: &str, _ip: Option) -> Result<(), Error> { + if self.fail_remove_ip { + return Err(anyhow::anyhow!("forced remove_ip failure").into()); + } + Ok(()) + } +} + +async fn set_ifcfg(nic: &SharedVirtualNic, ifcfg: RecordingIfConfiger) { + let mut device = nic.nic.lock().await; + device.set_ifname_for_test("et0".to_owned()); + device.set_ifcfg_for_test(Box::new(ifcfg)); +} + +#[tokio::test] +async fn conflicting_claims_leave_existing_configuration_intact() { + for configure_os in [true, false] { + let mut nic = SharedVirtualNic::new(virtual_nic_config(1500, None)); + let mut healthy = address_claims("10.144.1.1/24"); + healthy + .ipv6_addresses + .insert("fd00:1::1/64".parse().unwrap()); + nic.ifcfg.apply_member_claims(MEMBER, healthy.clone()); + let original = nic.ifcfg.effective(); + + let mut conflicting_ipv6 = address_claims("10.144.2.1/24"); + // IPv4 is disjoint, but the later IPv6 claim overlaps the healthy member. + conflicting_ipv6 + .ipv6_addresses + .insert("fd00:1::2/64".parse().unwrap()); + let conflicting_route = SharedIfConfigClaims { + ipv4_routes: BTreeSet::from([SharedIpv4Route::new( + "10.144.0.0".parse().unwrap(), + 16, + None, + )]), + ..Default::default() + }; + for claims in [conflicting_ipv6, conflicting_route] { + let rejected_member = uuid::Uuid::from_u128(2); + let err = nic + .apply_member_claims(rejected_member, claims, configure_os) + .await + .unwrap_err(); + assert!(err.to_string().contains("overlaps member")); + assert_eq!(nic.ifcfg.claims_of(MEMBER), healthy); + assert_eq!( + nic.ifcfg.claims_of(rejected_member), + SharedIfConfigClaims::default() + ); + assert!(nic.is_valid()); + } + assert_eq!(nic.ifcfg.effective(), original); + } +} + +#[tokio::test] +async fn mobile_claims_do_not_configure_the_os_interface() { + // There is no OS interface or ifname; native ifcfg calls would fail. + let mut nic = SharedVirtualNic::new(virtual_nic_config(1500, None)); + let claims = SharedIfConfigClaims { + ipv4_addresses: BTreeSet::from(["10.144.1.1/24".parse().unwrap()]), + ipv6_addresses: BTreeSet::from(["fd00:1::1/64".parse().unwrap()]), + ipv4_routes: BTreeSet::from([SharedIpv4Route::new( + "10.145.0.0".parse().unwrap(), + 16, + None, + )]), + ipv6_routes: BTreeSet::from([SharedIpv6Route::new("fd00:2::".parse().unwrap(), 64, None)]), + mtu: Some(1300), + }; + for claims in [claims, SharedIfConfigClaims::default()] { + nic.apply_member_claims(MEMBER, claims.clone(), false) + .await + .unwrap(); + assert_eq!(nic.ifcfg.claims_of(MEMBER), claims); + assert!(nic.is_valid()); + } + #[cfg(mobile)] + { + nic.member_registrations.insert(MEMBER, REGISTRATION); + nic.link_up().await.unwrap(); + nic.apply_member_mtu_for_registration(MEMBER, REGISTRATION, 1200) + .await + .unwrap(); + assert_eq!(nic.ifcfg.claims_of(MEMBER).mtu, Some(1200)); + } +} + +#[tokio::test] +async fn stale_registration_cannot_update_or_remove_current_claims() { + let mut nic = registered_nic(); + let original = nic.ifcfg.effective(); + let stale = uuid::Uuid::from_u128(9); + nic.apply_member_claims_for_registration(MEMBER, stale, SharedIfConfigClaims::default()) + .await + .unwrap(); + nic.apply_member_mtu_for_registration(MEMBER, stale, 1200) + .await + .unwrap(); + nic.remove_member_registration_claims(MEMBER, stale) + .await + .unwrap(); + assert_eq!(nic.ifcfg.effective(), original); + assert_eq!(nic.member_registrations.get(&MEMBER), Some(®ISTRATION)); +} + +#[cfg(not(mobile))] +#[tokio::test] +async fn failed_cleanup_and_replacement_invalidate_without_committing_claims() { + for replacing in [false, true] { + let mut nic = registered_nic(); + let original = nic.ifcfg.effective(); + set_ifcfg( + &nic, + RecordingIfConfiger { + fail_remove_ip: true, + ..Default::default() + }, + ) + .await; + let result = if replacing { + nic.attach_member_registration(MEMBER, uuid::Uuid::from_u128(11)) + .await + } else { + nic.remove_member_registration_claims(MEMBER, REGISTRATION) + .await + }; + assert!(result.is_err(), "replacing={replacing}"); + assert!(!nic.is_valid()); + assert_eq!(nic.ifcfg.effective(), original); + assert_eq!( + nic.member_registrations.get(&MEMBER).copied(), + replacing.then_some(REGISTRATION) + ); + } +} + +#[tokio::test] +async fn changed_route_source_is_removed_before_its_replacement_is_added() { + let route = SharedIpv4Route::new("10.90.2.0".parse().unwrap(), 24, Some(10)); + let fallback = "10.1.2.1".parse().unwrap(); + let preferred = "10.90.2.1".parse().unwrap(); + for adding_owner in [true, false] { + let mut nic = SharedVirtualNic::new(virtual_nic_config(1500, None)); + let ifcfg = RecordingIfConfiger::default(); + set_ifcfg(&nic, ifcfg.clone()).await; + let mut claims = address_claims("10.1.2.1/24"); + claims.ipv4_routes.insert(route.clone()); + nic.ifcfg.apply_member_claims(MEMBER, claims); + let mut with_owner = nic.ifcfg.clone(); + let mut claims = address_claims("10.90.2.1/24"); + claims.ipv4_routes.insert(route.clone()); + with_owner.apply_member_claims(uuid::Uuid::from_u128(2), claims); + let (next, old_source, new_source) = if adding_owner { + (with_owner, fallback, preferred) + } else { + ( + std::mem::replace(&mut nic.ifcfg, with_owner), + preferred, + fallback, + ) + }; + nic.apply_ifcfg_update(&next).await.unwrap(); + let changes = ifcfg.routes.lock().unwrap(); + let mut expected = vec![ + RouteChange::Remove(route.clone(), Some(old_source)), + RouteChange::Add(route.clone(), Some(new_source)), + ]; + if cfg!(target_os = "linux") && !adding_owner { + expected.push(RouteChange::Add(route.clone(), Some(new_source))); + } + assert_eq!(*changes, expected, "adding_owner={adding_owner}"); + } +} + +#[tokio::test] +async fn removing_route_preserves_its_old_cost_and_source_hint() { + let kept = SharedIpv4Route::new("10.90.4.0".parse().unwrap(), 24, Some(10)); + let removed = SharedIpv4Route { + cost: Some(20), + ..kept.clone() + }; + let source = "10.90.4.1".parse().unwrap(); + let mut nic = SharedVirtualNic::new(virtual_nic_config(1500, None)); + let ifcfg = RecordingIfConfiger::default(); + set_ifcfg(&nic, ifcfg.clone()).await; + let mut claims = address_claims("10.90.4.1/24"); + claims.ipv4_routes.extend([kept.clone(), removed.clone()]); + nic.ifcfg.apply_member_claims(MEMBER, claims.clone()); + claims.ipv4_routes.remove(&removed); + let mut next = nic.ifcfg.clone(); + next.apply_member_claims(MEMBER, claims); + nic.apply_ifcfg_update(&next).await.unwrap(); + assert_eq!( + *ifcfg.routes.lock().unwrap(), + [RouteChange::Remove(removed, Some(source))] + ); +} + +fn create_member( + registry: &mut SharedVirtualNicRegistry, + dev_name: &str, + mtu: u32, + netns: Option<&str>, +) -> SharedVirtualNicMember { + registry.create_member( + dev_name.to_owned(), + virtual_nic_config(mtu, netns), + uuid::Uuid::new_v4(), + Arc::new(Notify::new()), + ) +} + +#[test] +fn registry_shares_devices_only_within_the_same_name_and_namespace() { + let mut registry = SharedVirtualNicRegistry::new(); + let first = create_member(&mut registry, "et0", 1400, Some("net-a")); + for (name, netns, shared) in [ + ("et0", Some("net-a"), true), + ("et0", Some("net-b"), false), + ("et1", Some("net-a"), false), + ("et0", None, false), + ] { + let next = create_member(&mut registry, name, 1300, netns); + assert_eq!(Arc::ptr_eq(&first.shared_nic, &next.shared_nic), shared); + assert_eq!(next.configured_mtu, 1300); + } + assert_eq!(first.configured_mtu, 1400); +} + +#[test] +fn registry_replaces_invalid_devices() { + let mut registry = SharedVirtualNicRegistry::new(); + let first = create_member(&mut registry, "et0", 1500, None); + first.shared_nic.try_lock().unwrap().mark_invalid(); + assert!( + registry + .get("et0", &virtual_nic_config(1500, None)) + .is_none() + ); + let next = create_member(&mut registry, "et0", 1500, None); + assert!(!Arc::ptr_eq(&first.shared_nic, &next.shared_nic)); + assert!(Arc::ptr_eq( + ®istry + .get("et0", &virtual_nic_config(1500, None)) + .unwrap(), + &next.shared_nic, + )); +}