kumomta/crates/integration-tests/source.lua

local kumo = require 'kumo'
package.path = '../../assets/?.lua;' .. package.path

log_hooks = require 'policy-extras.log_hooks'

local TEST_DIR = os.getenv 'KUMOD_TEST_DIR'
local SINK_PORT = tonumber(os.getenv 'KUMOD_SMTP_SINK_PORT')
local WEBHOOK_PORT = os.getenv 'KUMOD_WEBHOOK_PORT'
local AMQPHOOK_URL = os.getenv 'KUMOD_AMQPHOOK_URL'
local AMQP_HOST_PORT = os.getenv 'KUMOD_AMQP_HOST_PORT'
local LISTENER_MAP = os.getenv 'KUMOD_LISTENER_DOMAIN_MAP'
local DEFERRED_SMTP_SERVER_MSG_INJECT =
  os.getenv 'KUMOD_DEFERRED_SMTP_SERVER_MSG_INJECT'
local BATCH_HANDLING = (os.getenv 'KUMOD_BATCH_HANDLING') or 'BifurcateAlways'
local MAX_RECIPIENTS_PER_BATCH = (os.getenv 'KUMOD_MAX_RECIPIENTS_PER_BATCH')
  or 100
local USE_SPLIT_TXN = os.getenv 'KUMOD_USE_SPLIT_TXN'
local INVALID_LINE_ENDINGS = os.getenv 'KUMOD_INVALID_LINE_ENDINGS'

kumo.on('init', function()
  kumo.configure_accounting_db_path(TEST_DIR .. '/accounting.db')
  kumo.aaa.configure_acct_log {
    log_dir = TEST_DIR .. '/acct',
    max_segment_duration = '1s',
  }

  local relay_hosts = { '0.0.0.0/0' }
  local RELAY_HOSTS = os.getenv 'KUMOD_RELAY_HOSTS'
  if RELAY_HOSTS then
    relay_hosts = kumo.json_parse(RELAY_HOSTS)
  end

  kumo.start_esmtp_listener {
    listen = '127.0.0.1:0',
    relay_hosts = relay_hosts,
    deferred_queue = (DEFERRED_SMTP_SERVER_MSG_INJECT and true) or false,
    peer = {
      ['127.0.0.1'] = {
        allow_xclient = true,
      },
    },
    via = {
      ['42.42.42.42'] = {
        banner = 'what do you get when you multiply six by nine?',
      },
    },
    batch_handling = BATCH_HANDLING,
    invalid_line_endings = INVALID_LINE_ENDINGS,
  }

  kumo.start_http_listener {
    listen = '127.0.0.1:0',
  }

  kumo.configure_local_logs {
    log_dir = TEST_DIR .. '/logs',
    max_segment_duration = '1s',
    headers = { 'X-*', 'Y-*', 'Subject' },
    meta = { 'xfer*' },
  }

  if WEBHOOK_PORT then
    kumo.configure_log_hook {
      name = 'webhook',
      headers = { 'Subject', 'X-*' },
    }
  end

  kumo.define_spool {
    name = 'data',
    path = TEST_DIR .. '/data-spool',
  }

  kumo.define_spool {
    name = 'meta',
    path = TEST_DIR .. '/meta-spool',
  }
end)

if AMQPHOOK_URL then
  log_hooks:new {
    name = 'amqp',
    constructor = function(domain, tenant, campaign)
      local sender = {}
      local client = kumo.amqp.build_client(AMQPHOOK_URL)

      function sender:send(msg)
        local result = client:publish_with_timeout({
          routing_key = 'woot',
          payload = msg:get_data(),
        }, 20000)

        if result.status == 'Ack' or result.status == 'NotRequested' then
          return string.format('250 %s', kumo.json_encode(result))
        end
        -- result.status must be `Nack`; log the full result
        kumo.reject(500, kumo.json_encode(result))
      end

      function sender:close()
        client:close()
      end

      return sender
    end,
  }
elseif AMQP_HOST_PORT then
  log_hooks:new {
    name = 'amqp',
    constructor = function(domain, tenant, campaign)
      local sender = {}
      local host, port = table.unpack(kumo.string.split(AMQP_HOST_PORT, ':'))

      function sender:send(msg)
        kumo.amqp.basic_publish {
          routing_key = 'woot',
          payload = msg:get_data(),
          connection = {
            host = host,
            port = tonumber(port),
          },
        }
        return '250 ok'
      end

      return sender
    end,
  }
end

if WEBHOOK_PORT then
  local min_batch_size = tonumber(os.getenv 'KUMOD_WEBHOOK_MIN_BATCH_SIZE')
  local max_batch_size = tonumber(os.getenv 'KUMOD_WEBHOOK_MAX_BATCH_SIZE')
  local max_batch_latency = os.getenv 'KUMOD_WEBHOOK_MAX_BATCH_LATENCY'
  if max_batch_size > 1 then
    log_hooks:new {
      name = 'webhookbatch',
      batch_size = max_batch_size,
      min_batch_size = min_batch_size,
      max_batch_latency = max_batch_latency,
      constructor = function(domain, tenant, campaign)
        local sender = {}

        local client = kumo.http.build_client {}
        function sender:send_batch(messages)
          local payload = {}
          for _, msg in ipairs(messages) do
            table.insert(payload, msg:get_meta 'log_record')
          end
          print(
            string.format(
              'batch size is %d *************** min=%d max=%d latency=%s',
              #payload,
              min_batch_size,
              max_batch_size,
              max_batch_latency
            )
          )
          local response = client
            :post(
              string.format('http://127.0.0.1:%d/log-batch', WEBHOOK_PORT)
            )
            :header('Content-Type', 'application/json')
            :body(kumo.serde.json_encode(payload))
            :send()

          local disposition = string.format(
            '%d %s: %s',
            response:status_code(),
            response:status_reason(),
            response:text()
          )

          if response:status_is_success() then
            return disposition
          end
          kumo.reject(500, disposition)
        end

        return sender
      end,
    }
  else
    kumo.on('should_enqueue_log_record', function(msg)
      local log_record = msg:get_meta 'log_record'
      -- avoid an infinite loop caused by logging that we logged
      if log_record.queue ~= 'webhook' then
        msg:set_meta('queue', 'webhook')
        return true
      end
      return false
    end)

    kumo.on('make.webhook', function(_domain, _tenant, _campaign)
      local sender = {}
      local client = kumo.http.build_client {}
      function sender:send(message)
        local response = client
          :post(string.format('http://127.0.0.1:%d/log', WEBHOOK_PORT))
          :header('Content-Type', 'application/json')
          :body(message:get_data())
          :send()

        local disposition = string.format(
          '%d %s: %s',
          response:status_code(),
          response:status_reason(),
          response:text()
        )

        if response:status_is_success() then
          return disposition
        end
        kumo.reject(500, disposition)
      end
      return sender
    end)
  end
end

kumo.on('get_listener_domain', function(domain, listener, conn_meta)
  if LISTENER_MAP then
    local map = kumo.json_parse(LISTENER_MAP)
    local params = map[domain]
    if params then
      return kumo.make_listener_domain(params)
    end
  end

  return kumo.make_listener_domain {
    relay_to = true,
    log_oob = true,
    log_arf = true,
  }
end)

if USE_SPLIT_TXN then
  kumo.on('smtp_server_split_transaction', function(msg, conn)
    -- This toy implementation batches by the first letter of
    -- the local part and the same domain. It is NOT an example
    -- of a valid production use case, it is merely exercising
    -- the logic to prove that it is functioning
    local batches = {}

    local keyed = {}
    for _, recip in ipairs(msg:recipient_list()) do
      local key = recip.user:sub(1, 1) .. '@' .. recip.domain:lower()
      if not keyed[key] then
        keyed[key] = {}
      end
      table.insert(keyed[key], recip)
    end

    for _, batch in pairs(keyed) do
      table.insert(batches, batch)
    end

    return batches
  end)
end

kumo.on('smtp_server_message_received', function(msg)
  local result = msg:import_scheduling_header 'X-Schedule'
  kumo.log_info('schedule result', kumo.serde.json_encode(result))
  -- This tenant header import is used by xfer.rs as a way to set
  -- the tenant metadata for the incoming message
  msg:import_x_headers { 'tenant' }
end)

kumo.on('get_queue_config', function(domain, tenant, campaign, routing_domain)
  kumo.log_warn('get_queue_config', domain, tenant, campaign, routing_domain)
  if domain == 'webhook' then
    return kumo.make_queue_config {
      protocol = {
        custom_lua = {
          constructor = 'make.webhook',
        },
      },
    }
  end

  local protocol = {
    -- Redirect traffic to the sink
    smtp = {
      mx_list = { 'localhost:' .. SINK_PORT },
    },
  }

  if domain == 'two-hosts.example.com' then
    protocol = {
      -- Redirect traffic to the sink, but with two hosts in the connection plan
      smtp = {
        mx_list = {
          { name = 'localhost-1', addr = '127.0.0.1:' .. SINK_PORT },
          { name = 'localhost-2', addr = '127.0.0.1:' .. SINK_PORT },
        },
      },
    }
  end

  -- This domain is coupled with the broken_first_choice_mx test
  if domain == 'broken-first-mx.example.com' then
    protocol = {
      -- Redirect traffic to the sink, with two hosts in the connection plan,
      -- but the first one is broken/not routable
      smtp = {
        mx_list = {
          -- 255.255.255.255 should not be reachable/routable on any system
          { name = 'brokenhost', addr = '255.255.255.255:' .. SINK_PORT },
          { name = 'workinghost', addr = '127.0.0.1:' .. SINK_PORT },
        },
      },
    }
  end

  if domain == 'rebind.example.com' then
    -- This domain is used by rebind_port.rs.
    -- In this case, we don't want the default sink route that was
    -- established above to take effect, we want to confirm that
    -- the port number embedded in either the domain or routing_domain
    -- are used, so we use the default smtp protocol settings here
    -- and rely on assertions in rebind_port.rs to validate the
    -- behavior.
    protocol = nil
  end

  return kumo.make_queue_config {
    protocol = protocol,
    retry_interval = os.getenv 'KUMOD_RETRY_INTERVAL',
    strategy = os.getenv 'KUMOD_QUEUE_STRATEGY',
    egress_pool = os.getenv 'KUMOD_POOL_NAME',
  }
end)

kumo.on('get_egress_pool', function(pool_name)
  if pool_name == 'warming' then
    -- coupled with source_selection_rate_pool.rs
    return kumo.make_egress_pool {
      name = pool_name,
      entries = {
        { name = 'warming_a' },
        { name = 'warming_b' },
      },
    }
  end

  error('integration-tests/source.lua: unhandled pool ' .. pool_name)
end)

kumo.on('get_egress_source', function(source_name)
  return kumo.make_egress_source {
    name = source_name,
  }
end)

kumo.on('get_egress_path_config', function(domain, source_name, _site_name)
  -- Allow sending to a sink
  local params = {
    enable_tls = os.getenv 'KUMOD_ENABLE_TLS' or 'OpportunisticInsecure',
    reconnect_strategy = os.getenv 'KUMOD_RECONNECT_STRATEGY'
      or 'ConnectNextHost',
    prohibited_hosts = {},
    opportunistic_tls_reconnect_on_failed_handshake = (
      (os.getenv 'KUMOD_OPPORTUNISTIC_TLS_RECONNECT') and true
    ) or false,
    source_selection_rate = os.getenv 'KUMOD_SOURCE_SELECTION_RATE',
    try_next_host_on_transport_error = (
      (os.getenv 'KUMOD_TRY_NEXT_HOST_ON_TRANSPORT_ERROR') and true
    ) or false,
    tls_prefer_openssl = ((os.getenv 'KUMOD_PREFER_OPENSSL') and true)
      or false,
    max_recipients_per_batch = tonumber(MAX_RECIPIENTS_PER_BATCH),

    -- Skip IPv6 addresses that come back for eg: localhost.
    -- For the most part the integration tests don't care about this,
    -- but the disconnect_XXX tests do make some assertions on the
    -- ordering, and in particular, disconnect_terminate_ok will be
    -- unhappy if the second address in its MX plan is unroutable IPv6.
    skip_hosts = { '::/0' },
  }

  if os.getenv 'KUMOD_CLIENT_CERTIFICATE' then
    params.tls_certificate = {
      key_data = os.getenv 'KUMOD_CLIENT_CERTIFICATE',
    }
  end
  if os.getenv 'KUMOD_CLIENT_PRIVATE_KEY' then
    params.tls_private_key = {
      key_data = os.getenv 'KUMOD_CLIENT_PRIVATE_KEY',
    }
  end

  -- See if there is a source-specific rate exported to us via the environment.
  -- We assign this using additional_source_selection_rates regardless of
  -- whether we have a more generate rate specified above so that we can
  -- excercise the additional_source_selection_rates collection logic in the core.
  local source_rate_name = 'KUMOD_SOURCE_SELECTION_RATE_'
    .. source_name:upper()
  local source_rate = os.getenv(source_rate_name)
  if source_rate then
    params.additional_source_selection_rates =
      { [source_rate_name] = source_rate }
  end

  local username = os.getenv 'KUMOD_SMTP_AUTH_USERNAME'
  local password = os.getenv 'KUMOD_SMTP_AUTH_PASSWORD'

  if username and password then
    params.smtp_auth_plain_username = username
    params.smtp_auth_plain_password = {
      key_data = password,
    }
    -- The test sink presents a self-signed certificate, so the TLS session is
    -- not validated; allow AUTH over it for these tests.
    params.allow_smtp_auth_plain_without_valid_certificate = true
  end

  if domain == 'two-hosts.example.com' then
    params.connection_limit = 1
  end

  if domain == 'webhookbatch.log_hook' then
    -- If we allow more connections, then we can end up
    -- with batches smaller than desired because we split
    -- them among multiple connections
    params.connection_limit = 1
  end

  kumo.log_warn('get_egress_path_config *******************', domain)

  return kumo.make_egress_path(params)
end)

if os.getenv 'KUMOD_WANT_REBIND' then
  kumo.on('rebind_message', function(message, data)
    message:append_header('X-Rebound', 'woot')
    message:set_meta('queue', data.queue)
  end)
end