From d7d066d493074b1d68deaba59c15165f96ac7bda Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Tue, 8 Aug 2023 15:19:24 +0100 Subject: [PATCH] proxy: delay auth on retry (#4929) ## Problem When an endpoint is shutting down, it can take a few seconds. Currently when starting a new compute, this causes an "endpoint is in transition" error. We need to add delays before retrying to ensure that we allow time for the endpoint to shutdown properly. ## Summary of changes Adds a delay before retrying in auth. connect_to_compute already has this delay --- proxy/src/auth/backend/classic.rs | 7 +++++-- proxy/src/proxy.rs | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/proxy/src/auth/backend/classic.rs b/proxy/src/auth/backend/classic.rs index d8ee1e8b64..46bd215f3b 100644 --- a/proxy/src/auth/backend/classic.rs +++ b/proxy/src/auth/backend/classic.rs @@ -5,7 +5,7 @@ use crate::{ auth::{self, AuthFlow, ClientCredentials}, compute, console::{self, AuthInfo, CachedNodeInfo, ConsoleReqExtra}, - proxy::handle_try_wake, + proxy::{handle_try_wake, retry_after}, sasl, scram, stream::PqStream, }; @@ -62,10 +62,13 @@ pub(super) async fn authenticate( } Ok(ControlFlow::Continue(e)) => { warn!(error = ?e, num_retries, retriable = true, "couldn't wake compute node"); - num_retries += 1; } Ok(ControlFlow::Break(n)) => break n, } + + let wait_duration = retry_after(num_retries); + num_retries += 1; + tokio::time::sleep(wait_duration).await; }; if let Some(keys) = scram_keys { use tokio_postgres::config::AuthKeys; diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index e6fcf901d9..0267d767ee 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -545,7 +545,7 @@ impl ShouldRetry for compute::ConnectionError { } } -fn retry_after(num_retries: u32) -> time::Duration { +pub fn retry_after(num_retries: u32) -> time::Duration { // 1.5 seems to be an ok growth factor heuristic BASE_RETRY_WAIT_DURATION.mul_f64(1.5_f64.powi(num_retries as i32)) }