Add support for custom key names in HashiCorp Vault secrets (#399)

* Add support for custom key names in HashiCorp Vault secrets

Co-authored-by: Wez Furlong <wez@wezfurlong.org>

crates/data-loader/src/lib.rs

@@ -20,9 +20,15 @@ pub enum KeySource {
         vault_token: Option<String>,
         vault_mount: String,
         vault_path: String,
+        #[serde(default = "default_vault_key")]
+        vault_key: String,
     },
 }
 
+fn default_vault_key() -> String {
+    "key".to_string()
+}
+
 #[cfg(feature = "impl")]
 impl KeySource {
     pub async fn get(&self) -> anyhow::Result<Vec<u8>> {
@@ -34,6 +40,7 @@ impl KeySource {
                 vault_token,
                 vault_mount,
                 vault_path,
+                vault_key,
             } => {
                 let address = match vault_address {
                     Some(a) => a.to_string(),
@@ -59,12 +66,7 @@ impl KeySource {
                         .build()?,
                 )?;
 
-                #[derive(Deserialize, Debug)]
-                struct Entry {
-                    key: String,
-                }
-
-                let entry: Entry = vaultrs::kv2::read(&client, vault_mount, vault_path)
+                let entry: serde_json::Value = vaultrs::kv2::read(&client, vault_mount, vault_path)
                     .await
                     .with_context(|| {
                         format!(
@@ -72,7 +74,14 @@ impl KeySource {
                         )
                     })?;
 
-                Ok(entry.key.into())
+                let value = entry
+                    .get(&vault_key)
+                    .and_then(|v| v.as_str())
+                    .ok_or_else(|| {
+                        anyhow!("vault secret at {vault_path} does not contain key '{vault_key}'")
+                    })?;
+
+                Ok(value.as_bytes().to_vec())
             }
         }
     }
@@ -252,6 +261,7 @@ mod test {
                 vault_token: Some(KEY.to_string()),
                 vault_mount: "secret".to_string(),
                 vault_path: path.to_string(),
+                vault_key: "key".to_string(),
             }
         }
     }
@@ -326,6 +336,36 @@ mod test {
 
         assert_eq!(pw, "bar");
 
+        // Test with a different key name
+        vault.put("custom_key", "custom_value").await?;
+
+        let source = KeySource::Vault {
+            vault_address: Some(vault.address()),
+            vault_token: Some(KEY.to_string()),
+            vault_mount: "secret".to_string(),
+            vault_path: "custom_key".to_string(),
+            vault_key: "custom_field".to_string(),
+        };
+
+        // This should fail because the vault secret has "key" but we're looking for "custom_field"
+        let result = source.get().await;
+        let err = result.unwrap_err();
+        assert!(err
+            .to_string()
+            .contains("vault secret at custom_key does not contain key 'custom_field'"));
+
+        // Test with the correct key name
+        let source = KeySource::Vault {
+            vault_address: Some(vault.address()),
+            vault_token: Some(KEY.to_string()),
+            vault_mount: "secret".to_string(),
+            vault_path: "custom_key".to_string(),
+            vault_key: "key".to_string(),
+        };
+
+        let data = source.get().await?;
+        assert_eq!(data, b"custom_value");
+
         Ok(())
     }
 }

docs/reference/keysource.md

@@ -80,13 +80,36 @@ local vault_signer = kumo.dkim.rsa_sha256_signer {
 
     -- vault_address = "http://127.0.0.1:8200"
     -- vault_token = "hvs.TOKENTOKENTOKEN"
+    
+    -- Optional: specify the key name within the vault secret
+    -- {{since('dev', inline=True)}}
+    -- Defaults to "key" if not specified
+    -- vault_key = "my_custom_key_name"
   },
 }
 ```
 
-The key must be stored as `key` under the `path` specified.
+The key must be stored under the `path` specified. By default, it looks for a field named `key` in the vault secret.
 For example, you might populate it like this:
 
 ```console
 $ vault kv put -mount=secret dkim/example.org key=@example-private-dkim-key.pem
 ```
+
+If you want to use a different field name, you can specify it with `vault_key` {{since('dev', inline=True)}}:
+
+```lua
+local vault_signer = kumo.dkim.rsa_sha256_signer {
+  key = {
+    vault_mount = 'secret',
+    vault_path = 'dkim/' .. msg:from_header().domain,
+    vault_key = 'private_key',  -- Look for 'private_key' instead of 'key'
+  },
+}
+```
+
+And store it in vault like this:
+
+```console
+$ vault kv put -mount=secret dkim/example.org private_key=@example-private-dkim-key.pem
+```

docs/reference/kumo.secrets/load.md

@@ -14,6 +14,8 @@ local request = kumo.http.build_client({}):get 'https://example.com/'
 local passwd = kumo.secrets.load {
   vault_mount = 'secret',
   vault_path = 'example.com-passwd',
+  -- Optional: specify a custom key name (defaults to "key")
+  -- vault_key = "password"
 }
 request:basic_auth('username', passwd)
 

docs/reference/kumo/make_egress_path/smtp_auth_plain_password.md

@@ -18,6 +18,8 @@ kumo.on('get_egress_path_config', function(domain, site_name)
     smtp_auth_plain_password = {
       vault_mount = 'secret',
       vault_path = 'smtp-auth/' .. domain,
+      -- Optional: specify a custom key name (defaults to "key")
+      -- vault_key = "password"
     },
   }
 end)

docs/reference/kumo/start_esmtp_listener/tls_certificate.md

@@ -27,6 +27,8 @@ kumo.start_esmtp_listener {
   tls_certificate = {
     vault_mount = 'secret',
     vault_path = 'tls/mail.example.com.cert',
+    -- Optional: specify a custom key name (defaults to "key")
+    -- vault_key = "certificate"
 
     -- Specify how to reach the vault; if you omit these,
     -- values will be read from $VAULT_ADDR and $VAULT_TOKEN
@@ -37,11 +39,30 @@ kumo.start_esmtp_listener {
 }
 ```
 
-The key must be stored as `key` (even though this is a certificate!) under the
-`path` specified.  For example, you might populate it like this:
+The certificate must be stored under the `path` specified. By default, it looks for a field named `key` in the vault secret.
+For example, you might populate it like this:
 
 ```
 $ vault kv put -mount=secret tls/mail.example.com.cert key=@mail.example.com.cert
 ```
 
+If you want to use a different field name, you can specify it with `vault_key` {{since('dev', inline=True)}}:
+
+```lua
+kumo.start_esmtp_listener {
+  -- ..
+  tls_certificate = {
+    vault_mount = 'secret',
+    vault_path = 'tls/mail.example.com.cert',
+    vault_key = 'certificate',  -- Look for 'certificate' instead of 'key'
+  },
+}
+```
+
+And store it in vault like this:
+
+```
+$ vault kv put -mount=secret tls/mail.example.com.cert certificate=@mail.example.com.cert
+```
+
 

docs/reference/kumo/start_esmtp_listener/tls_private_key.md

@@ -26,6 +26,8 @@ kumo.start_esmtp_listener {
   tls_private_key = {
     vault_mount = 'secret',
     vault_path = 'tls/mail.example.com.key',
+    -- Optional: specify a custom key name (defaults to "key")
+    -- vault_key = "private_key"
 
     -- Specify how to reach the vault; if you omit these,
     -- values will be read from $VAULT_ADDR and $VAULT_TOKEN
@@ -36,11 +38,30 @@ kumo.start_esmtp_listener {
 }
 ```
 
-The key must be stored as `key` under the `path` specified.
+The key must be stored under the `path` specified. By default, it looks for a field named `key` in the vault secret.
 For example, you might populate it like this:
 
 ```
 $ vault kv put -mount=secret tls/mail.example.com key=@mail.example.com.key
 ```
 
+If you want to use a different field name, you can specify it with `vault_key` {{since('dev', inline=True)}}:
+
+```lua
+kumo.start_esmtp_listener {
+  -- ..
+  tls_private_key = {
+    vault_mount = 'secret',
+    vault_path = 'tls/mail.example.com.key',
+    vault_key = 'private_key',  -- Look for 'private_key' instead of 'key'
+  },
+}
+```
+
+And store it in vault like this:
+
+```
+$ vault kv put -mount=secret tls/mail.example.com private_key=@mail.example.com.key
+```
+
 

docs/reference/kumo/start_http_listener/tls_certificate.md

@@ -20,6 +20,8 @@ kumo.start_http_listener {
   tls_certificate = {
     vault_mount = 'secret',
     vault_path = 'tls/mail.example.com.cert',
+    -- Optional: specify a custom key name (defaults to "key")
+    -- vault_key = "certificate"
 
     -- Specify how to reach the vault; if you omit these,
     -- values will be read from $VAULT_ADDR and $VAULT_TOKEN
@@ -30,11 +32,30 @@ kumo.start_http_listener {
 }
 ```
 
-The key must be stored as `key` (even though this is a certificate!) under the
-`path` specified.  For example, you might populate it like this:
+The certificate must be stored under the `path` specified. By default, it looks for a field named `key` in the vault secret.
+For example, you might populate it like this:
 
 ```
 $ vault kv put -mount=secret tls/mail.example.com.cert key=@mail.example.com.cert
 ```
 
+If you want to use a different field name, you can specify it with `vault_key` {{since('dev', inline=True)}}:
+
+```lua
+kumo.start_http_listener {
+  -- ..
+  tls_certificate = {
+    vault_mount = 'secret',
+    vault_path = 'tls/mail.example.com.cert',
+    vault_key = 'certificate',  -- Look for 'certificate' instead of 'key'
+  },
+}
+```
+
+And store it in vault like this:
+
+```
+$ vault kv put -mount=secret tls/mail.example.com.cert certificate=@mail.example.com.cert
+```
+
 

docs/reference/kumo/start_http_listener/tls_private_key.md

@@ -19,6 +19,8 @@ kumo.start_http_listener {
   tls_private_key = {
     vault_mount = 'secret',
     vault_path = 'tls/mail.example.com.key',
+    -- Optional: specify a custom key name (defaults to "key")
+    -- vault_key = "private_key"
 
     -- Specify how to reach the vault; if you omit these,
     -- values will be read from $VAULT_ADDR and $VAULT_TOKEN
@@ -29,11 +31,30 @@ kumo.start_http_listener {
 }
 ```
 
-The key must be stored as `key` under the `path` specified.
+The key must be stored under the `path` specified. By default, it looks for a field named `key` in the vault secret.
 For example, you might populate it like this:
 
 ```
 $ vault kv put -mount=secret tls/mail.example.com key=@mail.example.com.key
 ```
 
+If you want to use a different field name, you can specify it with `vault_key` {{since('dev', inline=True)}}:
+
+```lua
+kumo.start_http_listener {
+  -- ..
+  tls_private_key = {
+    vault_mount = 'secret',
+    vault_path = 'tls/mail.example.com.key',
+    vault_key = 'private_key',  -- Look for 'private_key' instead of 'key'
+  },
+}
+```
+
+And store it in vault like this:
+
+```
+$ vault kv put -mount=secret tls/mail.example.com private_key=@mail.example.com.key
+```
+
 

docs/userguide/configuration/dkim.md

@@ -183,6 +183,8 @@ at `/opt/kumomta/etc/dkim_data.toml` in this example.
 # of reading from disk
 vault_mount = "secret"
 vault_path_prefix = "dkim"
+# Optional: specify a custom key name (defaults to "key")
+# vault_key = "private_key"
 
 # To do double or triple signing, add each additional
 # signature name to this list and see the `signature."MyESPName"`

docs/userguide/configuration/httplisteners.md

@@ -46,6 +46,8 @@ kumo.start_http_listener {
    tls_certificate = {
     vault_mount = 'secret',
     vault_path = 'tls/mail.example.com.cert',
+    -- Optional: specify a custom key name (defaults to "key")
+    -- vault_key = "certificate"
     vault_address = "http://127.0.0.1:8200",
     vault_token = "hvs.TOKENTOKENTOKEN",
   },

docs/userguide/operation/outbound_auth.md

@@ -50,6 +50,8 @@ kumo.on(
       smtp_auth_plain_password = {
         vault_mount = 'secret',
         vault_path = 'smtp-auth/' .. routing_domain,
+        -- Optional: specify a custom key name (defaults to "key")
+        -- vault_key = "password"
       },
     }
   end
@@ -70,7 +72,7 @@ For example, to use a keysource with a local host, the following could be added
 ["192.168.1.10"]
 mx_rollup = false
 smtp_auth_plain_username = "daniel"
-smtp_auth_plain_password = { vault_mount = "secret", vault_path = "smtp-auth/local" }
+smtp_auth_plain_password = { vault_mount = "secret", vault_path = "smtp-auth/local", vault_key = "password" }
 {% endcall %}
 
 See the [traffic shaping](../configuration/trafficshaping.md#using-the-shapinglua-helper) section of the User Guide for additional information.

docs/userguide/policy/hashicorp_vault.md

@@ -17,6 +17,8 @@ local vault_signer = kumo.dkim.rsa_sha256_signer {
   key = {
     vault_mount = 'secret',
     vault_path = 'dkim/' .. msg:from_header().domain,
+    -- Optional: specify a custom key name (defaults to "key")
+    -- vault_key = "private_key"
     -- vault_address = "http://127.0.0.1:8200"
     -- vault_token = "hvs.TOKENTOKENTOKEN"
   },
@@ -73,6 +75,28 @@ vault kv put -mount=secret dkim/example.org key=@example-private-dkim-key.pem
 
 It is important to ensure you are storing Version-2 secrets with a "key=<value>" format. In the preceding example, the `key` points to a filename `example-private-dkim-key.pem`.
 
+## Using Custom Key Names
+{{since('dev')}}
+By default, KumoMTA looks for a field named `key` in your vault secret. If you want to use a different field name, you can specify it with the `vault_key` parameter:
+
+```lua
+local vault_signer = kumo.dkim.rsa_sha256_signer {
+  key = {
+    vault_mount = 'secret',
+    vault_path = 'dkim/' .. msg:from_header().domain,
+    vault_key = 'private_key',  -- Look for 'private_key' instead of 'key'
+  },
+}
+```
+
+And store it in vault like this:
+
+```bash
+vault kv put -mount=secret dkim/example.org private_key=@example-private-dkim-key.pem
+```
+
+This is particularly useful when you have multiple keys stored in the same vault secret or when your vault secrets follow a different naming convention.
+
 ## Ways to Use Vault With KumoMTA
 
 Vault has a number of advantages over statically storing secrets. Aside from the obvious security benefits of not exposing your passwords and security keys in your code, it also allows you to *physically* separate the information. One key use case is storing the vault server in a private network while the KumoMTA instances are deployed around the world or in public colocation or cloud services. If a remote server is compromised, the local vault server can be secured to prevent data leakage.

docs/userguide/policy/inbound_auth.md

@@ -64,6 +64,8 @@ function vault_auth_check(user, password)
     == kumo.secrets.load {
       vault_mount = 'secret',
       vault_path = 'smtp-auth/' .. user,
+      -- Optional: specify a custom key name (defaults to "key")
+      -- vault_key = "password"
     }
 end
 

simple_policy.lua

@@ -274,7 +274,9 @@ local function common_processing(msg)
       --[[
       key = {
         vault_mount = "secret",
-        vault_path = "dkim/" .. msg:sender().domain
+        vault_path = "dkim/" .. msg:sender().domain,
+        -- Optional: specify a custom key name (defaults to "key")
+        -- vault_key = "private_key"
       }
       ]]
     }