From 96bcfba79e4919a7a5b8fddd2149231b42059883 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Krzysztof=20Szafra=C5=84ski?= Date: Fri, 18 Jul 2025 12:17:58 +0200 Subject: [PATCH] [proxy] Cache GetEndpointAccessControl errors (#12571) Related to https://github.com/neondatabase/cloud/issues/19353 --- proxy/src/cache/project_info.rs | 286 +++++++++++++++--- .../control_plane/client/cplane_proxy_v1.rs | 185 ++++++----- proxy/src/control_plane/errors.rs | 2 +- proxy/src/control_plane/messages.rs | 16 +- proxy/src/control_plane/mod.rs | 6 +- 5 files changed, 376 insertions(+), 119 deletions(-) diff --git a/proxy/src/cache/project_info.rs b/proxy/src/cache/project_info.rs index c812779e30..0ef09a8a9a 100644 --- a/proxy/src/cache/project_info.rs +++ b/proxy/src/cache/project_info.rs @@ -10,6 +10,7 @@ use tokio::time::Instant; use tracing::{debug, info}; use crate::config::ProjectInfoCacheOptions; +use crate::control_plane::messages::{ControlPlaneErrorMessage, Reason}; use crate::control_plane::{EndpointAccessControl, RoleAccessControl}; use crate::intern::{AccountIdInt, EndpointIdInt, ProjectIdInt, RoleNameInt}; use crate::types::{EndpointId, RoleName}; @@ -36,22 +37,37 @@ impl Entry { } pub(crate) fn get(&self) -> Option<&T> { - (self.expires_at > Instant::now()).then_some(&self.value) + (!self.is_expired()).then_some(&self.value) + } + + fn is_expired(&self) -> bool { + self.expires_at <= Instant::now() } } struct EndpointInfo { - role_controls: HashMap>, - controls: Option>, + role_controls: HashMap>>, + controls: Option>>, } +type ControlPlaneResult = Result>; + impl EndpointInfo { - pub(crate) fn get_role_secret(&self, role_name: RoleNameInt) -> Option { - self.role_controls.get(&role_name)?.get().cloned() + pub(crate) fn get_role_secret_with_ttl( + &self, + role_name: RoleNameInt, + ) -> Option<(ControlPlaneResult, Duration)> { + let entry = self.role_controls.get(&role_name)?; + let ttl = entry.expires_at - Instant::now(); + Some((entry.get()?.clone(), ttl)) } - pub(crate) fn get_controls(&self) -> Option { - self.controls.as_ref()?.get().cloned() + pub(crate) fn get_controls_with_ttl( + &self, + ) -> Option<(ControlPlaneResult, Duration)> { + let entry = self.controls.as_ref()?; + let ttl = entry.expires_at - Instant::now(); + Some((entry.get()?.clone(), ttl)) } pub(crate) fn invalidate_endpoint(&mut self) { @@ -153,28 +169,28 @@ impl ProjectInfoCacheImpl { self.cache.get(&endpoint_id) } - pub(crate) fn get_role_secret( + pub(crate) fn get_role_secret_with_ttl( &self, endpoint_id: &EndpointId, role_name: &RoleName, - ) -> Option { + ) -> Option<(ControlPlaneResult, Duration)> { let role_name = RoleNameInt::get(role_name)?; let endpoint_info = self.get_endpoint_cache(endpoint_id)?; - endpoint_info.get_role_secret(role_name) + endpoint_info.get_role_secret_with_ttl(role_name) } - pub(crate) fn get_endpoint_access( + pub(crate) fn get_endpoint_access_with_ttl( &self, endpoint_id: &EndpointId, - ) -> Option { + ) -> Option<(ControlPlaneResult, Duration)> { let endpoint_info = self.get_endpoint_cache(endpoint_id)?; - endpoint_info.get_controls() + endpoint_info.get_controls_with_ttl() } pub(crate) fn insert_endpoint_access( &self, account_id: Option, - project_id: ProjectIdInt, + project_id: Option, endpoint_id: EndpointIdInt, role_name: RoleNameInt, controls: EndpointAccessControl, @@ -183,26 +199,89 @@ impl ProjectInfoCacheImpl { if let Some(account_id) = account_id { self.insert_account2endpoint(account_id, endpoint_id); } - self.insert_project2endpoint(project_id, endpoint_id); + if let Some(project_id) = project_id { + self.insert_project2endpoint(project_id, endpoint_id); + } if self.cache.len() >= self.config.size { // If there are too many entries, wait until the next gc cycle. return; } - let controls = Entry::new(controls, self.config.ttl); - let role_controls = Entry::new(role_controls, self.config.ttl); + debug!( + key = &*endpoint_id, + "created a cache entry for endpoint access" + ); + + let controls = Some(Entry::new(Ok(controls), self.config.ttl)); + let role_controls = Entry::new(Ok(role_controls), self.config.ttl); match self.cache.entry(endpoint_id) { clashmap::Entry::Vacant(e) => { e.insert(EndpointInfo { role_controls: HashMap::from_iter([(role_name, role_controls)]), - controls: Some(controls), + controls, }); } clashmap::Entry::Occupied(mut e) => { let ep = e.get_mut(); - ep.controls = Some(controls); + ep.controls = controls; + if ep.role_controls.len() < self.config.max_roles { + ep.role_controls.insert(role_name, role_controls); + } + } + } + } + + pub(crate) fn insert_endpoint_access_err( + &self, + endpoint_id: EndpointIdInt, + role_name: RoleNameInt, + msg: Box, + ttl: Option, + ) { + if self.cache.len() >= self.config.size { + // If there are too many entries, wait until the next gc cycle. + return; + } + + debug!( + key = &*endpoint_id, + "created a cache entry for an endpoint access error" + ); + + let ttl = ttl.unwrap_or(self.config.ttl); + + let controls = if msg.get_reason() == Reason::RoleProtected { + // RoleProtected is the only role-specific error that control plane can give us. + // If a given role name does not exist, it still returns a successful response, + // just with an empty secret. + None + } else { + // We can cache all the other errors in EndpointInfo.controls, + // because they don't depend on what role name we pass to control plane. + Some(Entry::new(Err(msg.clone()), ttl)) + }; + + let role_controls = Entry::new(Err(msg), ttl); + + match self.cache.entry(endpoint_id) { + clashmap::Entry::Vacant(e) => { + e.insert(EndpointInfo { + role_controls: HashMap::from_iter([(role_name, role_controls)]), + controls, + }); + } + clashmap::Entry::Occupied(mut e) => { + let ep = e.get_mut(); + if let Some(entry) = &ep.controls + && !entry.is_expired() + && entry.value.is_ok() + { + // If we have cached non-expired, non-error controls, keep them. + } else { + ep.controls = controls; + } if ep.role_controls.len() < self.config.max_roles { ep.role_controls.insert(role_name, role_controls); } @@ -245,7 +324,7 @@ impl ProjectInfoCacheImpl { return; }; - if role_controls.get().expires_at <= Instant::now() { + if role_controls.get().is_expired() { role_controls.remove(); } } @@ -284,13 +363,11 @@ impl ProjectInfoCacheImpl { #[cfg(test)] mod tests { - use std::sync::Arc; - use super::*; - use crate::control_plane::messages::EndpointRateLimitConfig; + use crate::control_plane::messages::{Details, EndpointRateLimitConfig, ErrorInfo, Status}; use crate::control_plane::{AccessBlockerFlags, AuthSecret}; use crate::scram::ServerSecret; - use crate::types::ProjectId; + use std::sync::Arc; #[tokio::test] async fn test_project_info_cache_settings() { @@ -301,9 +378,9 @@ mod tests { ttl: Duration::from_secs(1), gc_interval: Duration::from_secs(600), }); - let project_id: ProjectId = "project".into(); + let project_id: Option = Some(ProjectIdInt::from(&"project".into())); let endpoint_id: EndpointId = "endpoint".into(); - let account_id: Option = None; + let account_id = None; let user1: RoleName = "user1".into(); let user2: RoleName = "user2".into(); @@ -316,7 +393,7 @@ mod tests { cache.insert_endpoint_access( account_id, - (&project_id).into(), + project_id, (&endpoint_id).into(), (&user1).into(), EndpointAccessControl { @@ -332,7 +409,7 @@ mod tests { cache.insert_endpoint_access( account_id, - (&project_id).into(), + project_id, (&endpoint_id).into(), (&user2).into(), EndpointAccessControl { @@ -346,11 +423,17 @@ mod tests { }, ); - let cached = cache.get_role_secret(&endpoint_id, &user1).unwrap(); - assert_eq!(cached.secret, secret1); + let (cached, ttl) = cache + .get_role_secret_with_ttl(&endpoint_id, &user1) + .unwrap(); + assert_eq!(cached.unwrap().secret, secret1); + assert_eq!(ttl, cache.config.ttl); - let cached = cache.get_role_secret(&endpoint_id, &user2).unwrap(); - assert_eq!(cached.secret, secret2); + let (cached, ttl) = cache + .get_role_secret_with_ttl(&endpoint_id, &user2) + .unwrap(); + assert_eq!(cached.unwrap().secret, secret2); + assert_eq!(ttl, cache.config.ttl); // Shouldn't add more than 2 roles. let user3: RoleName = "user3".into(); @@ -358,7 +441,7 @@ mod tests { cache.insert_endpoint_access( account_id, - (&project_id).into(), + project_id, (&endpoint_id).into(), (&user3).into(), EndpointAccessControl { @@ -372,17 +455,144 @@ mod tests { }, ); - assert!(cache.get_role_secret(&endpoint_id, &user3).is_none()); + assert!( + cache + .get_role_secret_with_ttl(&endpoint_id, &user3) + .is_none() + ); - let cached = cache.get_endpoint_access(&endpoint_id).unwrap(); + let cached = cache + .get_endpoint_access_with_ttl(&endpoint_id) + .unwrap() + .0 + .unwrap(); assert_eq!(cached.allowed_ips, allowed_ips); tokio::time::advance(Duration::from_secs(2)).await; - let cached = cache.get_role_secret(&endpoint_id, &user1); + let cached = cache.get_role_secret_with_ttl(&endpoint_id, &user1); assert!(cached.is_none()); - let cached = cache.get_role_secret(&endpoint_id, &user2); + let cached = cache.get_role_secret_with_ttl(&endpoint_id, &user2); assert!(cached.is_none()); - let cached = cache.get_endpoint_access(&endpoint_id); + let cached = cache.get_endpoint_access_with_ttl(&endpoint_id); assert!(cached.is_none()); } + + #[tokio::test] + async fn test_caching_project_info_errors() { + let cache = ProjectInfoCacheImpl::new(ProjectInfoCacheOptions { + size: 10, + max_roles: 10, + ttl: Duration::from_secs(1), + gc_interval: Duration::from_secs(600), + }); + let project_id = Some(ProjectIdInt::from(&"project".into())); + let endpoint_id: EndpointId = "endpoint".into(); + let account_id = None; + + let user1: RoleName = "user1".into(); + let user2: RoleName = "user2".into(); + let secret = Some(AuthSecret::Scram(ServerSecret::mock([1; 32]))); + + let role_msg = Box::new(ControlPlaneErrorMessage { + error: "role is protected and cannot be used for password-based authentication" + .to_owned() + .into_boxed_str(), + http_status_code: http::StatusCode::NOT_FOUND, + status: Some(Status { + code: "PERMISSION_DENIED".to_owned().into_boxed_str(), + message: "role is protected and cannot be used for password-based authentication" + .to_owned() + .into_boxed_str(), + details: Details { + error_info: Some(ErrorInfo { + reason: Reason::RoleProtected, + }), + retry_info: None, + user_facing_message: None, + }, + }), + }); + + let generic_msg = Box::new(ControlPlaneErrorMessage { + error: "oh noes".to_owned().into_boxed_str(), + http_status_code: http::StatusCode::NOT_FOUND, + status: None, + }); + + let get_role_secret = |endpoint_id, role_name| { + cache + .get_role_secret_with_ttl(endpoint_id, role_name) + .unwrap() + .0 + }; + let get_endpoint_access = + |endpoint_id| cache.get_endpoint_access_with_ttl(endpoint_id).unwrap().0; + + // stores role-specific errors only for get_role_secret + cache.insert_endpoint_access_err( + (&endpoint_id).into(), + (&user1).into(), + role_msg.clone(), + None, + ); + assert_eq!( + get_role_secret(&endpoint_id, &user1).unwrap_err().error, + role_msg.error + ); + assert!(cache.get_endpoint_access_with_ttl(&endpoint_id).is_none()); + + // stores non-role specific errors for both get_role_secret and get_endpoint_access + cache.insert_endpoint_access_err( + (&endpoint_id).into(), + (&user1).into(), + generic_msg.clone(), + None, + ); + assert_eq!( + get_role_secret(&endpoint_id, &user1).unwrap_err().error, + generic_msg.error + ); + assert_eq!( + get_endpoint_access(&endpoint_id).unwrap_err().error, + generic_msg.error + ); + + // error isn't returned for other roles in the same endpoint + assert!( + cache + .get_role_secret_with_ttl(&endpoint_id, &user2) + .is_none() + ); + + // success for a role does not overwrite errors for other roles + cache.insert_endpoint_access( + account_id, + project_id, + (&endpoint_id).into(), + (&user2).into(), + EndpointAccessControl { + allowed_ips: Arc::new(vec![]), + allowed_vpce: Arc::new(vec![]), + flags: AccessBlockerFlags::default(), + rate_limits: EndpointRateLimitConfig::default(), + }, + RoleAccessControl { + secret: secret.clone(), + }, + ); + assert!(get_role_secret(&endpoint_id, &user1).is_err()); + assert!(get_role_secret(&endpoint_id, &user2).is_ok()); + // ...but does clear the access control error + assert!(get_endpoint_access(&endpoint_id).is_ok()); + + // storing an error does not overwrite successful access control response + cache.insert_endpoint_access_err( + (&endpoint_id).into(), + (&user2).into(), + generic_msg.clone(), + None, + ); + assert!(get_role_secret(&endpoint_id, &user2).is_err()); + assert!(get_endpoint_access(&endpoint_id).is_ok()); + } } diff --git a/proxy/src/control_plane/client/cplane_proxy_v1.rs b/proxy/src/control_plane/client/cplane_proxy_v1.rs index bb785b8b0c..8a0403c0b0 100644 --- a/proxy/src/control_plane/client/cplane_proxy_v1.rs +++ b/proxy/src/control_plane/client/cplane_proxy_v1.rs @@ -68,6 +68,66 @@ impl NeonControlPlaneClient { self.endpoint.url().as_str() } + async fn get_and_cache_auth_info( + &self, + ctx: &RequestContext, + endpoint: &EndpointId, + role: &RoleName, + cache_key: &EndpointId, + extract: impl FnOnce(&EndpointAccessControl, &RoleAccessControl) -> T, + ) -> Result { + match self.do_get_auth_req(ctx, endpoint, role).await { + Ok(auth_info) => { + let control = EndpointAccessControl { + allowed_ips: Arc::new(auth_info.allowed_ips), + allowed_vpce: Arc::new(auth_info.allowed_vpc_endpoint_ids), + flags: auth_info.access_blocker_flags, + rate_limits: auth_info.rate_limits, + }; + let role_control = RoleAccessControl { + secret: auth_info.secret, + }; + let res = extract(&control, &role_control); + + self.caches.project_info.insert_endpoint_access( + auth_info.account_id, + auth_info.project_id, + cache_key.into(), + role.into(), + control, + role_control, + ); + + if let Some(project_id) = auth_info.project_id { + ctx.set_project_id(project_id); + } + + Ok(res) + } + Err(err) => match err { + GetAuthInfoError::ApiError(ControlPlaneError::Message(ref msg)) => { + let retry_info = msg.status.as_ref().and_then(|s| s.details.retry_info); + + // If we can retry this error, do not cache it, + // unless we were given a retry delay. + if msg.could_retry() && retry_info.is_none() { + return Err(err); + } + + self.caches.project_info.insert_endpoint_access_err( + cache_key.into(), + role.into(), + msg.clone(), + retry_info.map(|r| Duration::from_millis(r.retry_delay_ms)), + ); + + Err(err) + } + err => Err(err), + }, + } + } + async fn do_get_auth_req( &self, ctx: &RequestContext, @@ -284,43 +344,34 @@ impl super::ControlPlaneApi for NeonControlPlaneClient { ctx: &RequestContext, endpoint: &EndpointId, role: &RoleName, - ) -> Result { - let normalized_ep = &endpoint.normalize(); - if let Some(secret) = self + ) -> Result { + let key = endpoint.normalize(); + + if let Some((role_control, ttl)) = self .caches .project_info - .get_role_secret(normalized_ep, role) + .get_role_secret_with_ttl(&key, role) { - return Ok(secret); + return match role_control { + Err(mut msg) => { + info!(key = &*key, "found cached get_role_access_control error"); + + // if retry_delay_ms is set change it to the remaining TTL + replace_retry_delay_ms(&mut msg, |_| ttl.as_millis() as u64); + + Err(GetAuthInfoError::ApiError(ControlPlaneError::Message(msg))) + } + Ok(role_control) => { + debug!(key = &*key, "found cached role access control"); + Ok(role_control) + } + }; } - let auth_info = self.do_get_auth_req(ctx, endpoint, role).await?; - - let control = EndpointAccessControl { - allowed_ips: Arc::new(auth_info.allowed_ips), - allowed_vpce: Arc::new(auth_info.allowed_vpc_endpoint_ids), - flags: auth_info.access_blocker_flags, - rate_limits: auth_info.rate_limits, - }; - let role_control = RoleAccessControl { - secret: auth_info.secret, - }; - - if let Some(project_id) = auth_info.project_id { - let normalized_ep_int = normalized_ep.into(); - - self.caches.project_info.insert_endpoint_access( - auth_info.account_id, - project_id, - normalized_ep_int, - role.into(), - control, - role_control.clone(), - ); - ctx.set_project_id(project_id); - } - - Ok(role_control) + self.get_and_cache_auth_info(ctx, endpoint, role, &key, |_, role_control| { + role_control.clone() + }) + .await } #[tracing::instrument(skip_all)] @@ -330,38 +381,30 @@ impl super::ControlPlaneApi for NeonControlPlaneClient { endpoint: &EndpointId, role: &RoleName, ) -> Result { - let normalized_ep = &endpoint.normalize(); - if let Some(control) = self.caches.project_info.get_endpoint_access(normalized_ep) { - return Ok(control); + let key = endpoint.normalize(); + + if let Some((control, ttl)) = self.caches.project_info.get_endpoint_access_with_ttl(&key) { + return match control { + Err(mut msg) => { + info!( + key = &*key, + "found cached get_endpoint_access_control error" + ); + + // if retry_delay_ms is set change it to the remaining TTL + replace_retry_delay_ms(&mut msg, |_| ttl.as_millis() as u64); + + Err(GetAuthInfoError::ApiError(ControlPlaneError::Message(msg))) + } + Ok(control) => { + debug!(key = &*key, "found cached endpoint access control"); + Ok(control) + } + }; } - let auth_info = self.do_get_auth_req(ctx, endpoint, role).await?; - - let control = EndpointAccessControl { - allowed_ips: Arc::new(auth_info.allowed_ips), - allowed_vpce: Arc::new(auth_info.allowed_vpc_endpoint_ids), - flags: auth_info.access_blocker_flags, - rate_limits: auth_info.rate_limits, - }; - let role_control = RoleAccessControl { - secret: auth_info.secret, - }; - - if let Some(project_id) = auth_info.project_id { - let normalized_ep_int = normalized_ep.into(); - - self.caches.project_info.insert_endpoint_access( - auth_info.account_id, - project_id, - normalized_ep_int, - role.into(), - control.clone(), - role_control, - ); - ctx.set_project_id(project_id); - } - - Ok(control) + self.get_and_cache_auth_info(ctx, endpoint, role, &key, |control, _| control.clone()) + .await } #[tracing::instrument(skip_all)] @@ -390,13 +433,9 @@ impl super::ControlPlaneApi for NeonControlPlaneClient { info!(key = &*key, "found cached wake_compute error"); // if retry_delay_ms is set, reduce it by the amount of time it spent in cache - if let Some(status) = &mut msg.status { - if let Some(retry_info) = &mut status.details.retry_info { - retry_info.retry_delay_ms = retry_info - .retry_delay_ms - .saturating_sub(created_at.elapsed().as_millis() as u64) - } - } + replace_retry_delay_ms(&mut msg, |delay| { + delay.saturating_sub(created_at.elapsed().as_millis() as u64) + }); Err(WakeComputeError::ControlPlane(ControlPlaneError::Message( msg, @@ -478,6 +517,14 @@ impl super::ControlPlaneApi for NeonControlPlaneClient { } } +fn replace_retry_delay_ms(msg: &mut ControlPlaneErrorMessage, f: impl FnOnce(u64) -> u64) { + if let Some(status) = &mut msg.status + && let Some(retry_info) = &mut status.details.retry_info + { + retry_info.retry_delay_ms = f(retry_info.retry_delay_ms); + } +} + /// Parse http response body, taking status code into account. fn parse_body serde::Deserialize<'a>>( status: StatusCode, diff --git a/proxy/src/control_plane/errors.rs b/proxy/src/control_plane/errors.rs index 12843e48c7..1e43010957 100644 --- a/proxy/src/control_plane/errors.rs +++ b/proxy/src/control_plane/errors.rs @@ -52,7 +52,7 @@ impl ReportableError for ControlPlaneError { | Reason::EndpointNotFound | Reason::EndpointDisabled | Reason::BranchNotFound - | Reason::InvalidEphemeralEndpointOptions => ErrorKind::User, + | Reason::WrongLsnOrTimestamp => ErrorKind::User, Reason::RateLimitExceeded => ErrorKind::ServiceRateLimit, diff --git a/proxy/src/control_plane/messages.rs b/proxy/src/control_plane/messages.rs index cf193ed268..d44d7efcc3 100644 --- a/proxy/src/control_plane/messages.rs +++ b/proxy/src/control_plane/messages.rs @@ -107,7 +107,7 @@ pub(crate) struct ErrorInfo { // Schema could also have `metadata` field, but it's not structured. Skip it for now. } -#[derive(Clone, Copy, Debug, Deserialize, Default)] +#[derive(Clone, Copy, Debug, Deserialize, Default, PartialEq, Eq)] pub(crate) enum Reason { /// RoleProtected indicates that the role is protected and the attempted operation is not permitted on protected roles. #[serde(rename = "ROLE_PROTECTED")] @@ -133,9 +133,9 @@ pub(crate) enum Reason { /// or that the subject doesn't have enough permissions to access the requested branch. #[serde(rename = "BRANCH_NOT_FOUND")] BranchNotFound, - /// InvalidEphemeralEndpointOptions indicates that the specified LSN or timestamp are wrong. - #[serde(rename = "INVALID_EPHEMERAL_OPTIONS")] - InvalidEphemeralEndpointOptions, + /// WrongLsnOrTimestamp indicates that the specified LSN or timestamp are wrong. + #[serde(rename = "WRONG_LSN_OR_TIMESTAMP")] + WrongLsnOrTimestamp, /// RateLimitExceeded indicates that the rate limit for the operation has been exceeded. #[serde(rename = "RATE_LIMIT_EXCEEDED")] RateLimitExceeded, @@ -205,7 +205,7 @@ impl Reason { | Reason::EndpointNotFound | Reason::EndpointDisabled | Reason::BranchNotFound - | Reason::InvalidEphemeralEndpointOptions => false, + | Reason::WrongLsnOrTimestamp => false, // we were asked to go away Reason::RateLimitExceeded | Reason::NonDefaultBranchComputeTimeExceeded @@ -257,19 +257,19 @@ pub(crate) struct GetEndpointAccessControl { pub(crate) rate_limits: EndpointRateLimitConfig, } -#[derive(Copy, Clone, Deserialize, Default)] +#[derive(Copy, Clone, Deserialize, Default, Debug)] pub struct EndpointRateLimitConfig { pub connection_attempts: ConnectionAttemptsLimit, } -#[derive(Copy, Clone, Deserialize, Default)] +#[derive(Copy, Clone, Deserialize, Default, Debug)] pub struct ConnectionAttemptsLimit { pub tcp: Option, pub ws: Option, pub http: Option, } -#[derive(Copy, Clone, Deserialize)] +#[derive(Copy, Clone, Deserialize, Debug)] pub struct LeakyBucketSetting { pub rps: f64, pub burst: f64, diff --git a/proxy/src/control_plane/mod.rs b/proxy/src/control_plane/mod.rs index a8c59dad0c..9bbd3f4fb7 100644 --- a/proxy/src/control_plane/mod.rs +++ b/proxy/src/control_plane/mod.rs @@ -82,7 +82,7 @@ impl NodeInfo { } } -#[derive(Copy, Clone, Default)] +#[derive(Copy, Clone, Default, Debug)] pub(crate) struct AccessBlockerFlags { pub public_access_blocked: bool, pub vpc_access_blocked: bool, @@ -92,12 +92,12 @@ pub(crate) type NodeInfoCache = TimedLru>>; pub(crate) type CachedNodeInfo = Cached<&'static NodeInfoCache, NodeInfo>; -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct RoleAccessControl { pub secret: Option, } -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct EndpointAccessControl { pub allowed_ips: Arc>, pub allowed_vpce: Arc>,