data-loader: allow callback/event based loading of secrets

This allows a (more or less) arbitrary lua function to be used to load
data.

The primary advantage of this is that the size of the resulting
KeySource struct is smaller than it would be to hold the actual data
inline.  It also enables the surrounding code to be slightly better
factored.

crates/config/src/lib.rs

@@ -195,6 +195,20 @@ impl LuaConfig {
             .set("_KUMO_CURRENT_EVENT", name.to_string())
     }
 
+    /// Convert an array of args into a MultiValue that can be passed
+    /// to a callback signature
+    pub fn convert_args_to_multi<A: Serialize>(
+        &self,
+        args: &[A],
+    ) -> anyhow::Result<mlua::MultiValue> {
+        let lua = self.inner.as_ref().unwrap();
+        let mut arg_vec = vec![];
+        for a in args.iter() {
+            arg_vec.push(lua.lua.to_value(a)?);
+        }
+        Ok(mlua::MultiValue::from_vec(arg_vec))
+    }
+
     /// Intended to be used together with kumo.spawn_task
     pub async fn convert_args_and_call_callback<A: Serialize>(
         &mut self,

crates/data-loader/src/lib.rs

@@ -24,6 +24,10 @@ pub enum KeySource {
         #[serde(default = "default_vault_key")]
         vault_key: String,
     },
+    Event {
+        event_name: String,
+        event_args: Vec<serde_json::Value>,
+    },
 }
 
 fn default_vault_key() -> String {
@@ -84,6 +88,20 @@ impl KeySource {
 
                 Ok(value.as_bytes().to_vec())
             }
+            Self::Event {
+                event_name,
+                event_args,
+            } => {
+                let mut config = config::load_config().await?;
+                let sig = config::CallbackSignature::<mlua::MultiValue, mlua::String>::new(
+                    event_name.clone(),
+                );
+
+                let args = config.convert_args_to_multi(event_args)?;
+                let result = config.async_call_callback_non_default(&sig, args).await?;
+
+                Ok(result.as_bytes().to_vec())
+            }
         }
     }
 }

crates/data-loader/test-event.lua

@@ -0,0 +1,21 @@
+local kumo = require 'kumo'
+package.path = 'assets/?.lua;' .. package.path
+local utils = require 'policy-extras.policy_utils'
+
+kumo.on('loader-example-1', function(one, two)
+  return string.format('called-with-%s-%s', one, two)
+end)
+
+kumo.on('main', function()
+  local text = kumo.secrets.load {
+    event_name = 'loader-example-1',
+    event_args = { 'a', 'b' },
+  }
+  utils.assert_eq(text, 'called-with-a-b')
+
+  local text = kumo.secrets.load {
+    event_name = 'loader-example-1',
+    event_args = { 1, 'b' },
+  }
+  utils.assert_eq(text, 'called-with-1-b')
+end)

crates/run-lua-tests/src/main.rs

@@ -122,11 +122,28 @@ end)
     }
 
     async fn run_crate_test(test_file: &str) -> anyhow::Result<TestResult> {
-        let mut script_file = NamedTempFile::new().context("failed to make temp file")?;
+        let content = tokio::fs::read_to_string(test_file).await?;
+        if content.contains("kumo.on('main'") {
+            // The script is self-contained and can be run directly
+            Self::run_test_script(test_file, test_file).await
+        } else {
+            if content.contains("kumo.on(") {
+                // Hmm, ideally the sanity check that is in kumo.on itself would
+                // suffice for this, but it special cases the `main` callback handler,
+                // which we use below, to exclude it from its overall check.
+                // That exclusion is required for some of the config validation
+                // handling logic to work appropriately, so let's not touch that
+                // other code just now and do a sanity check here that is suitable
+                // for our test harness.
+                anyhow::bail!(
+                    "{test_file} attempts to use kumo.on but needs to define kumo.on('kumo.main') for that to work"
+                );
+            }
+            let mut script_file = NamedTempFile::new().context("failed to make temp file")?;
 
-        write!(
-            script_file,
-            r#"
+            write!(
+                script_file,
+                r#"
 local kumo = require 'kumo'
 package.path = 'assets/?.lua;' .. package.path
 
@@ -134,16 +151,17 @@ kumo.on('main', function()
   dofile "{test_file}"
 end)
 "#
-        )?;
+            )?;
 
-        Self::run_test_script(
-            script_file
-                .path()
-                .to_str()
-                .context("temp file is not utf8")?,
-            test_file,
-        )
-        .await
+            Self::run_test_script(
+                script_file
+                    .path()
+                    .to_str()
+                    .context("temp file is not utf8")?,
+                test_file,
+            )
+            .await
+        }
     }
 
     async fn run_test_script(script: &str, test_file: &str) -> anyhow::Result<TestResult> {

docs/changelog/main.md

@@ -89,3 +89,6 @@
  * [keysource](../reference/keysource.md) now supports inline binary bytes
    being passed via `key_data`.  Previously, only UTF-8 strings could be
    passed that way.
+ * [keysource](../reference/keysource.md) now supports callback/event based
+   data loading, which is similar to inline `key_data`, but allows for more
+   efficient cache keys that use less RAM.

docs/reference/keysource.md

@@ -24,6 +24,9 @@ local file_signer = kumo.dkim.rsa_sha256_signer {
 
 ### Caller Provided Data
 
+!!! note
+    Please also take a look at the callback/event based example further below
+
 When the value is a table with the field `key_data`,
 the value of the `key_data` field will be used as the key
 data when needed:
@@ -113,3 +116,45 @@ And store it in vault like this:
 ```console
 $ vault kv put -mount=secret dkim/example.org private_key=@example-private-dkim-key.pem
 ```
+
+### Callback/Event based Data Source
+
+{{since('dev')}}
+
+You may define an event handling function to perform parameterized data
+fetching.  This is in some ways similar to using the `key_data` form described
+above, but has the advantage of allowing deferred rather than eager access to
+the data.
+
+For example, you could enhance the sqlite dkim example above to use this form;
+the advantage is that the cache keys become smaller and more efficient, and the
+key source object is then a bit more declarative:
+
+```lua
+-- Define the event handling callback. The name must match up to the
+-- event_name field used in the keysource object below
+kumo.on('fetch-my-domain-key', function(domain, selector)
+  local db = sqlite:open '/opt/kumomta/etc/dkim/keys.db'
+  local result = db:execute(
+    'select data from keys where domain=? and selector=?',
+    domain,
+    selector
+  )
+  -- The callback must return a string which, depending on
+  -- the code that is consuming the keysource, may be binary.
+  -- DKIM keys are PEM encoded ASCII.
+  return result[1]
+end)
+
+local sqlite_signer = kumo.dkim.rsa_sha256_signer {
+  key = {
+    -- name of the event handling callback you
+    -- have registered via kumo.on.
+    event_name = 'fetch-my-domain-key',
+    -- The list of parameters that will be passed to
+    -- the event handling callback
+    event_args = { msg:from_header().domain, 'default' },
+  },
+}
+```
+