update certificate files in the watch task

compute_tools/src/compute.rs

@@ -2198,8 +2198,29 @@ impl ComputeNode {
             "TLS certificates found"
         );
 
+        // ensure the keys are saved before continuing.
+        let key_pair = crate::tls::load_certs_blocking(&tls_config);
+        while let Err(e) =
+            crate::tls::update_key_path_blocking(Path::new(&self.params.pgdata), &key_pair)
+        {
+            error!("could not save TLS certificates: {e}");
+            std::thread::sleep(Duration::from_millis(20));
+        }
+
         tokio::task::spawn_blocking(move || {
             'cert_update: loop {
+                // wait for a new certificate update
+                digest = crate::tls::wait_until_cert_changed(digest, &tls_config.cert_path);
+
+                // ensure the keys are saved before continuing.
+                let key_pair = crate::tls::load_certs_blocking(&tls_config);
+                while let Err(e) =
+                    crate::tls::update_key_path_blocking(Path::new(&self.params.pgdata), &key_pair)
+                {
+                    error!("could not save TLS certificates: {e}");
+                    std::thread::sleep(Duration::from_millis(20));
+                }
+
                 // let postgres/pgbouncer/local_proxy know the new cert/key exists.
                 // we need to wait until it's configurable first.
 
@@ -2234,9 +2255,6 @@ impl ComputeNode {
                 }
                 drop(state);
 
-                // wait for a new certificate update
-                digest = crate::tls::wait_until_cert_changed(digest, &tls_config.cert_path);
-
                 info!(
                     cert_path = tls_config.cert_path,
                     key_path = tls_config.key_path,

compute_tools/src/config.rs

@@ -16,7 +16,7 @@ use crate::pg_helpers::{
     DatabricksSettingsExt as _, GenericOptionExt, GenericOptionsSearch, PgOptionsSerialize,
     escape_conf_value,
 };
-use crate::tls::{self, SERVER_CRT, SERVER_KEY};
+use crate::tls::{SERVER_CRT, SERVER_KEY};
 
 use utils::shard::{ShardIndex, ShardNumber};
 
@@ -178,15 +178,9 @@ pub fn write_postgres_conf(
     }
 
     // tls
-    if let Some(tls_config) = tls_config {
+    if tls_config.is_some() {
         writeln!(file, "ssl = on")?;
 
-        // postgres requires the keyfile to be in a secure file,
-        // currently too complicated to ensure that at the VM level,
-        // so we just copy them to another file instead. :shrug:
-        let keys = tls::load_certs_blocking(tls_config);
-        tls::update_key_path_blocking(pgdata_path, &keys)?;
-
         // these are the default, but good to be explicit.
         writeln!(file, "ssl_cert_file = '{SERVER_CRT}'")?;
         writeln!(file, "ssl_key_file = '{SERVER_KEY}'")?;

compute_tools/src/pg_helpers.rs

@@ -523,17 +523,6 @@ pub async fn tune_pgbouncer(
     tls_config: Option<TlsConfig>,
 ) -> Result<()> {
     if let Some(tls_config) = tls_config {
-        // pgbouncer starts in a half-ok state if it cannot find these files.
-        // It will default to client_tls_sslmode=deny, which causes proxy to error.
-        // There is a small window at startup where these files don't yet exist in the VM.
-        // Best to wait until it exists.
-        loop {
-            if let Ok(true) = tokio::fs::try_exists(&tls_config.key_path).await {
-                break;
-            }
-            tokio::time::sleep(Duration::from_millis(500)).await
-        }
-
         pgbouncer_config.insert("client_tls_cert_file".to_string(), tls_config.cert_path);
         pgbouncer_config.insert("client_tls_key_file".to_string(), tls_config.key_path);
         pgbouncer_config.insert("client_tls_sslmode".to_string(), "allow".to_string());