perf(common): cache interface snapshots on the Windows bind path (#2744)

* perf(common): cache interface snapshots on the Windows bind path

Each bind with BindDev::Auto resolved the interface name via a full
NetworkInterface::show() enumeration, and on Windows setup_socket_for_win
then resolved the same name to an index through a second enumeration.
With many adapters (VMware / Hyper-V / APIPA leftovers are common on
Windows), one connect round turned into O(candidates) synchronous
enumerations on the async worker: issue #2735 measured ~4.9s of
candidate preparation against a 2s connect budget, and even on a healthy
8-adapter machine the loop cost ~124ms for 10 candidates.

Reuse a snapshot of the interface list with a 1s TTL in both places.
Measured on an 11-candidate machine: candidate bind loop drops from
~146ms to ~0.5ms; on machines with slow enumeration the win is
proportionally larger, keeping candidate preparation inside the connect
budget. Complements #2736, which offloads the remaining syscalls to the
blocking pool.

* fix(network): coalesce interface refreshes and limit caching to Windows

* fix(windows): isolate bind cache and correct lookup diagnostics
This commit is contained in:
fanyang
2026-10-27 00:38:53 +08:00
committed by GitHub
parent 3463190838
commit b974acd737
3 changed files with 200 additions and 6 deletions
+16 -1
View File
@@ -64,6 +64,7 @@ pub fn disable_connection_reset<S: AsRawSocket>(socket: &S) -> io::Result<()> {
Ok(())
}
// Configuration operations must see newly created or recreated adapters immediately.
pub fn find_interface_index(iface_name: &str) -> io::Result<u32> {
let ifaces = network_interface::NetworkInterface::show().map_err(|e| {
io::Error::new(
@@ -81,8 +82,22 @@ pub fn find_interface_index(iface_name: &str) -> io::Result<u32> {
))
}
fn find_interface_index_for_bind(iface_name: &str) -> io::Result<u32> {
let ifaces = crate::common::network::get_interfaces_cached().ok_or_else(|| {
io::Error::new(
io::ErrorKind::NotFound,
format!("Failed to get interfaces for {}", iface_name),
)
})?;
ifaces
.iter()
.find(|iface| iface.name == iface_name)
.map(|iface| iface.index)
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, iface_name.to_string()))
}
pub fn set_ip_unicast_if(socket: SOCKET, addr: &SocketAddr, iface: &str) -> io::Result<()> {
let if_index = find_interface_index(iface)?;
let if_index = find_interface_index_for_bind(iface)?;
unsafe {
// https://docs.microsoft.com/en-us/windows/win32/winsock/ipproto-ip-socket-options
+103
View File
@@ -1,6 +1,10 @@
#[cfg(not(mobile))]
use std::collections::HashMap;
use std::net::IpAddr;
#[cfg(any(target_os = "windows", test))]
use std::sync::{Arc, RwLock};
#[cfg(any(target_os = "windows", test))]
use std::time::{Duration, Instant};
use network_interface::{NetworkInterface, NetworkInterfaceConfig};
#[cfg(all(target_os = "macos", not(feature = "macos-ne")))]
@@ -303,6 +307,48 @@ pub(crate) async fn collect_interfaces(net_ns: NetNS, filter: bool) -> Vec<Netwo
}
}
/// The Windows bind path resolves an interface name and then its index
/// through full `NetworkInterface::show()` enumerations for every bind
/// candidate. With 10+ adapters (VMware / Hyper-V / APIPA leftovers are
/// common), one connect round turns into O(candidates) synchronous
/// enumerations on the async worker. Interfaces rarely change within a
/// connect round, so snapshots are reused for a short TTL.
/// Only use this cache for socket binding; interface configuration needs
/// fresh enumeration to observe adapter creation and replacement immediately.
#[cfg(target_os = "windows")]
pub(crate) fn get_interfaces_cached() -> Option<Arc<Vec<NetworkInterface>>> {
static SNAPSHOT: InterfaceSnapshot = RwLock::new(None);
get_or_refresh_interfaces(&SNAPSHOT, || NetworkInterface::show().ok())
}
#[cfg(any(target_os = "windows", test))]
type InterfaceSnapshot = RwLock<Option<(Instant, Arc<Vec<NetworkInterface>>)>>;
#[cfg(any(target_os = "windows", test))]
fn get_or_refresh_interfaces(
snapshot: &InterfaceSnapshot,
enumerate: impl FnOnce() -> Option<Vec<NetworkInterface>>,
) -> Option<Arc<Vec<NetworkInterface>>> {
const SNAPSHOT_TTL: Duration = Duration::from_secs(1);
if let Some((at, ifaces)) = snapshot.read().ok().and_then(|guard| guard.clone())
&& at.elapsed() < SNAPSHOT_TTL
{
return Some(ifaces);
}
// Recheck after acquiring the write lock: another caller may have refreshed
// while we waited. Keep the lock during enumeration to coalesce refreshes.
let mut snapshot = snapshot.write().ok()?;
if let Some((at, ifaces)) = snapshot.as_ref()
&& at.elapsed() < SNAPSHOT_TTL
{
return Some(ifaces.clone());
}
let ifaces = Arc::new(enumerate()?);
*snapshot = Some((Instant::now(), ifaces.clone()));
Some(ifaces)
}
#[cfg(feature = "faketcp")]
fn convert_pnet_interface(iface: pnet_datalink::NetworkInterface) -> NetworkInterface {
let internal = iface.is_loopback();
@@ -498,3 +544,60 @@ mod tests {
assert_eq!(before, after);
}
}
#[cfg(test)]
mod interface_snapshot_tests {
use super::*;
use std::sync::{
Barrier,
atomic::{AtomicUsize, Ordering},
};
#[test]
fn concurrent_refreshes_share_cold_and_expired_snapshots() {
for initial in [
None,
Some((Instant::now() - Duration::from_secs(2), Arc::new(vec![]))),
] {
let snapshot = InterfaceSnapshot::new(initial);
let calls = AtomicUsize::new(0);
let barrier = Barrier::new(8);
let results = std::thread::scope(|scope| {
let handles: Vec<_> = (0..8)
.map(|_| {
scope.spawn(|| {
barrier.wait();
get_or_refresh_interfaces(&snapshot, || {
calls.fetch_add(1, Ordering::SeqCst);
std::thread::sleep(Duration::from_millis(20));
Some(vec![])
})
.unwrap()
})
})
.collect();
handles
.into_iter()
.map(|h| h.join().unwrap())
.collect::<Vec<_>>()
});
assert_eq!(calls.load(Ordering::SeqCst), 1);
for result in &results {
assert!(Arc::ptr_eq(&results[0], result));
}
}
}
#[test]
fn failed_refresh_is_retried_without_serving_expired_snapshot() {
let snapshot = InterfaceSnapshot::new(Some((
Instant::now() - Duration::from_secs(2),
Arc::new(vec![]),
)));
assert!(get_or_refresh_interfaces(&snapshot, || None).is_none());
let refreshed = get_or_refresh_interfaces(&snapshot, || Some(vec![])).unwrap();
let cached =
get_or_refresh_interfaces(&snapshot, || panic!("cache should be fresh")).unwrap();
assert!(Arc::ptr_eq(&refreshed, &cached));
}
}
+81 -5
View File
@@ -1,5 +1,4 @@
use bon::builder;
use network_interface::NetworkInterfaceConfig as _;
use std::net::{IpAddr, SocketAddr};
use crate::common::netns::NetNS;
@@ -10,11 +9,18 @@ pub(crate) fn get_interface_name_by_ip(local_ip: &IpAddr) -> Option<String> {
if local_ip.is_unspecified() || local_ip.is_multicast() {
return None;
}
let ifaces = network_interface::NetworkInterface::show().ok()?;
for iface in ifaces {
for addr in iface.addr {
#[cfg(target_os = "windows")]
let ifaces = crate::common::network::get_interfaces_cached()?;
// Enumerate under the caller's namespace guard on non-Windows platforms.
#[cfg(not(target_os = "windows"))]
let ifaces = {
use network_interface::NetworkInterfaceConfig;
network_interface::NetworkInterface::show().ok()?
};
for iface in ifaces.iter() {
for addr in &iface.addr {
if addr.ip() == *local_ip {
return Some(iface.name);
return Some(iface.name.clone());
}
}
}
@@ -244,6 +250,8 @@ pub async fn bind<B: Bindable>(
#[cfg(test)]
pub(crate) mod tests {
use atomic_shim::AtomicU64;
#[cfg(target_os = "windows")]
use std::time::Duration;
use std::{sync::Arc, time::Instant};
use anyhow::Context as _;
@@ -261,6 +269,74 @@ pub(crate) mod tests {
use crate::common::netns::NetNS;
#[cfg(target_os = "windows")]
#[test]
fn interface_snapshot_lookup_smoke_test() {
// The process-global cache may expire between reads. Identity checks
// belong to the isolated helper tests in common::network instead.
for _ in 0..2 {
let ifaces = crate::common::network::get_interfaces_cached()
.expect("interface lookup should work");
assert!(!ifaces.is_empty());
}
}
// Diagnostic probe for the issue #2735 slow candidate loop: run with
// `cargo test -p easytier --lib probe_interface_lookup -- --ignored --nocapture`.
// Compare only IP-to-name and name-to-index lookups, not socket binding.
#[cfg(target_os = "windows")]
#[test]
#[ignore = "diagnostic probe; enumerates live Windows adapters"]
fn probe_interface_lookup_latency_with_and_without_cache() {
use network_interface::{NetworkInterface, NetworkInterfaceConfig};
use std::net::IpAddr;
type EnumerateInterfaces = fn() -> Option<Arc<Vec<NetworkInterface>>>;
fn lookup_index(ip: IpAddr, enumerate: EnumerateInterfaces) -> Option<u32> {
let ifaces = enumerate()?;
let name = &ifaces
.iter()
.find(|iface| iface.addr.iter().any(|addr| addr.ip() == ip))?
.name;
let ifaces = enumerate()?;
ifaces
.iter()
.find(|iface| iface.name == *name)
.map(|iface| iface.index)
}
let ifaces = NetworkInterface::show().unwrap();
let candidates: Vec<_> = ifaces
.iter()
.flat_map(|iface| {
iface.addr.iter().filter_map(move |addr| {
matches!(addr.ip(), IpAddr::V4(_)).then(|| (iface.name.clone(), addr.ip()))
})
})
.collect();
println!("IPv4 lookup candidates ({}):", candidates.len());
let providers: [(&str, EnumerateInterfaces); 2] = [
("uncached", || NetworkInterface::show().ok().map(Arc::new)),
("cached", crate::common::network::get_interfaces_cached),
];
for (label, enumerate) in providers {
let mut total = Duration::ZERO;
for (name, ip) in &candidates {
let start = Instant::now();
let index = lookup_index(*ip, enumerate);
let elapsed = start.elapsed();
total += elapsed;
println!(" [{label}] {ip:>16} ({name}): lookup {elapsed:?}, index={index:?}");
}
println!(
"{label} lookup total: {total:?} ({} candidates)",
candidates.len()
);
}
}
#[tokio::test]
async fn bind_rejects_occupied_ports() {
for ip in ["127.0.0.1", "::1"] {