diff --git a/Makefile b/Makefile index 7f49bd70..48a4d12d 100644 --- a/Makefile +++ b/Makefile @@ -16,9 +16,7 @@ fc: RUSTFLAGS="--cfg tokio_unstable -D warnings" cargo fc check --fail-fast test-lua: - ./assets/run-lua-test assets/policy-extras/typing.lua - ./assets/run-lua-test assets/policy-extras/queue.lua - ./assets/run-lua-test assets/policy-extras/listener_domains.lua + ./assets/run-lua-test test: build test-lua ./docs/update-openapi.sh diff --git a/assets/policy-extras/listener_domains.lua b/assets/policy-extras/listener_domains.lua index d2a9bbd3..760f73b0 100644 --- a/assets/policy-extras/listener_domains.lua +++ b/assets/policy-extras/listener_domains.lua @@ -234,13 +234,6 @@ function mod:setup(data_files) return get_listener_domain end ---[[ -Run some basic unit tests for the data parsing/merging; use it like this: - -``` -KUMOMTA_RUN_UNIT_TESTS=1 ./target/debug/kumod --policy assets/policy-extras/listener_domains.lua -``` -]] function mod:test() local open_relay = [=[ ['*'] @@ -293,10 +286,4 @@ log_oob = false ) end -if os.getenv 'KUMOMTA_RUN_UNIT_TESTS' then - kumo.configure_accounting_db_path(os.tmpname()) - mod:test() - os.exit(0) -end - return mod diff --git a/assets/policy-extras/queue.lua b/assets/policy-extras/queue.lua index 4e96ccf4..99057e5d 100644 --- a/assets/policy-extras/queue.lua +++ b/assets/policy-extras/queue.lua @@ -1,6 +1,7 @@ local mod = {} local kumo = require 'kumo' local utils = require 'policy-extras.policy_utils' +local typing = require 'policy-extras.typing' --[[ Usage: @@ -67,49 +68,6 @@ end) --- ]] -local function parse_one(data) - local options = {} - local tenants = {} - local queues = {} - - for k, v in pairs(data) do - if k == 'tenant' then - for tenant_name, tenant_options in pairs(v) do - local tenant = tenants[tenant_name] or {} - utils.merge_into(tenant_options, tenant) - tenants[tenant_name] = tenant - end - elseif k == 'queue' then - for domain_name, queue_options in pairs(v) do - local domain_options = queues[domain_name] or {} - utils.recursive_merge_into(queue_options, domain_options) - queues[domain_name] = domain_options - end - else - options[k] = v - end - end - - local result = { - options = options, - tenants = tenants, - queues = queues, - } - - return result -end - -local function merge_data(loaded_files, no_compile) - local result = {} - for _, data in ipairs(loaded_files) do - utils.recursive_merge_into(parse_one(data), result) - end - if not no_compile then - result.queues = kumo.domain_map.new(result.queues) - end - return result -end - -- Return true if the `name` and `value` pair provided -- is a valid queue configuration option. -- It does this by asking the internal rust code to validate it. @@ -119,6 +77,141 @@ local function is_queue_config_option(name, value) return status end +local Bool, List, Map, Option, Record, String = + typing.boolean, + typing.list, + typing.map, + typing.option, + typing.record, + typing.string + +local CampaignConfig = Record('CampaignConfig', { + _dynamic = is_queue_config_option, + overall_max_message_rate = Option(String), + routing_domain = Option(String), +}) + +local TenantWithCampaignConfig = Record('TenantWithCampaignConfig', { + _dynamic = is_queue_config_option, + overall_max_message_rate = Option(String), + routing_domain = Option(String), + + campaigns = Option(Map(String, CampaignConfig)), +}) + +local DomainConfig = Record('DomainConfig', { + _dynamic = is_queue_config_option, + overall_max_message_rate = Option(String), + routing_domain = Option(String), + + tenants = Option(Map(String, TenantWithCampaignConfig)), +}) + +local TenantConfig = Record('TenantConfig', { + campaigns = Option(Map(String, CampaignConfig)), + _dynamic = is_queue_config_option, + require_authz = Option(List(String)), + overall_max_message_rate = Option(String), +}) + +local QueueHelperConfig = Record('QueueHelperConfig', { + scheduling_header = Option(String), + tenant_header = Option(String), + remove_tenant_header = Option(Bool), + campaign_header = Option(String), + remove_campaign_header = Option(Bool), + default_tenant = Option(String), + + tenants = Option(Map(String, TenantConfig)), + queues = Option(Map(String, DomainConfig)), +}) + +local function parse_tenant_with_campaign(data) + local tenant = TenantWithCampaignConfig { + campaigns = {}, + } + -- print('parse_tenant_with_campaign', kumo.serde.json_encode_pretty(data)) + for k, v in pairs(data) do + -- Since we allow an arbitrary key to map to a campaign, and a mixture + -- of known-to-lua and known-to-rust options, short of replicating + -- the logic in the typing module here, the best option is just to + -- try to assign it, and if it raises an error, then it must be + -- a campaign value. + local status, result = pcall(function() + tenant[k] = v + end) + if not status then + tenant.campaigns[k] = v + end + end + + return tenant +end + +local function parse_domain(data) + local domain = DomainConfig { + tenants = {}, + } + -- print('parse_domain', kumo.serde.json_encode_pretty(data)) + for k, v in pairs(data) do + -- Since we allow an arbitrary key to map to a tenant, and a mixture + -- of known-to-lua and known-to-rust options, short of replicating + -- the logic in the typing module here, the best option is just to + -- try to assign it, and if it raises an error, then it must be + -- a tenant value. + local status, result = pcall(function() + domain[k] = v + end) + if not status then + domain.tenants[k] = parse_tenant_with_campaign(v) + end + end + + return domain +end + +local function parse_config(data) + local result = QueueHelperConfig { + tenants = {}, + queues = {}, + } + + for k, v in pairs(data) do + if k == 'tenant' then + for tenant_name, tenant_options in pairs(v) do + local tenant = result.tenants[tenant_name] or {} + utils.merge_into(tenant_options, tenant) + result.tenants[tenant_name] = tenant + end + elseif k == 'queue' then + for domain_name, queue_options in pairs(v) do + local domain_options = result.queues[domain_name] or {} + utils.recursive_merge_into( + parse_domain(queue_options), + domain_options + ) + result.queues[domain_name] = domain_options + end + else + result[k] = v + end + end + + return result +end + +local function merge_data(loaded_files, no_compile) + local result = QueueHelperConfig {} + for _, data in ipairs(loaded_files) do + utils.recursive_merge_into(parse_config(data), result) + end + -- print(kumo.json_encode_pretty(result)) + if not no_compile then + result.queues = kumo.domain_map.new(result.queues) + end + return result +end + local function load_queue_config(file_names, no_compile) local data = {} for _, file_name in ipairs(file_names) do @@ -166,7 +259,7 @@ local function resolve_config(data, domain, tenant, campaign, allow_all) end if domain_config then - local tenant_config = domain_config[tenant] + local tenant_config = domain_config.tenants[tenant] if tenant_config then for k, v in pairs(tenant_config) do if allow_all or is_queue_config_option(k, v) then @@ -174,7 +267,7 @@ local function resolve_config(data, domain, tenant, campaign, allow_all) end end - local campaign = tenant_config[campaign] + local campaign = tenant_config.campaigns[campaign] if campaign then for k, v in pairs(campaign) do @@ -217,32 +310,31 @@ local function resolve_overall_throttle_specs( end local function apply_impl(msg, data) - if data.options.scheduling_header then - msg:import_scheduling_header(data.options.scheduling_header, true) + if data.scheduling_header then + msg:import_scheduling_header(data.scheduling_header, true) end - if data.options.campaign_header then - local campaign = - msg:get_first_named_header_value(data.options.campaign_header) + if data.campaign_header then + local campaign = msg:get_first_named_header_value(data.campaign_header) if campaign then msg:set_meta('campaign', campaign) - if data.options.remove_campaign_header then - msg:remove_all_named_headers(data.options.campaign_header) + if data.remove_campaign_header then + msg:remove_all_named_headers(data.campaign_header) end end end local tenant = nil local tenant_source = nil - if data.options.tenant_header then - tenant = msg:get_first_named_header_value(data.options.tenant_header) + if data.tenant_header then + tenant = msg:get_first_named_header_value(data.tenant_header) if tenant then - tenant_source = string.format("'%s' header", data.options.tenant_header) - if data.options.remove_tenant_header then - msg:remove_all_named_headers(data.options.tenant_header) + tenant_source = string.format("'%s' header", data.tenant_header) + if data.remove_tenant_header then + msg:remove_all_named_headers(data.tenant_header) end end end - if not tenant and data.options.default_tenant then - tenant = data.options.default_tenant + if not tenant and data.default_tenant then + tenant = data.default_tenant tenant_source = 'default_tenant option' end if tenant then @@ -507,13 +599,6 @@ You should adjust the initialization order so that queue.lua is last. end end) ---[[ -Run some basic unit tests for the data parsing/merging; use it like this: - -``` -KUMOMTA_RUN_UNIT_TESTS=1 ./target/debug/kumod --policy assets/policy-extras/queue.lua -``` -]] function mod:test() local base_data = [=[ default_tenant = 'mytenant' @@ -605,10 +690,4 @@ egress_pool = "bar" utils.assert_eq(msg:get_meta 'routing_domain', '[10.0.0.1]') end -if os.getenv 'KUMOMTA_RUN_UNIT_TESTS' then - kumo.configure_accounting_db_path(os.tmpname()) - mod:test() - os.exit(0) -end - return mod diff --git a/assets/policy-extras/typing.lua b/assets/policy-extras/typing.lua index 4353815e..101ece35 100644 --- a/assets/policy-extras/typing.lua +++ b/assets/policy-extras/typing.lua @@ -166,6 +166,19 @@ local function casting_validate_value(ty, other) end -- Returns a record constructor +-- +-- fields is a map, where the key is the field +-- name and the value is the type of that field. +-- +-- _dynamic is a special entry: it enables performing a runtime +-- check to see if a given key/value pair is a valid value for +-- assignment. This is used to outsource validation to native +-- rust code in the embedding application. The value in that +-- case must be a function that accepts the key and value +-- and returns a boolean to indicate whether that combination +-- is acceptable. +-- +-- { _dynamic = function(key, value) return true end } function mod.record(name, fields) local ty = { name = name, @@ -190,6 +203,12 @@ function mod.record(name, fields) function ty.__newindex(t, k, v) local field_def = ty.fields[k] if not field_def then + local dyn_validate = ty.fields._dynamic + if dyn_validate and dyn_validate(k, v) then + rawset(t, k, v) + return + end + TypeError:new(string.format("%s: unknown field '%s'", ty.name, k)) :raise() end @@ -216,19 +235,21 @@ function mod.record(name, fields) for k, def in pairs(ty.fields) do if not obj[k] then - if def.default_value then - obj[k] = def.default_value - elseif not def.is_optional then - TypeError - :new( - string.format( - "%s: missing value for field '%s' of type '%s'", - ty.name, - k, - def.name + if type(def) == 'table' then + if def.default_value then + obj[k] = def.default_value + elseif not def.is_optional then + TypeError + :new( + string.format( + "%s: missing value for field '%s' of type '%s'", + ty.name, + k, + def.name + ) ) - ) - :raise() + :raise() + end end end end @@ -591,10 +612,4 @@ function mod:test() ) end -if os.getenv 'KUMOMTA_RUN_UNIT_TESTS' then - kumo.configure_accounting_db_path(os.tmpname()) - mod:test() - os.exit(0) -end - return mod diff --git a/assets/run-lua-test b/assets/run-lua-test index d312e317..a0895ac7 100755 --- a/assets/run-lua-test +++ b/assets/run-lua-test @@ -1,21 +1,41 @@ #!/bin/bash # This is a little helper script that is used to run unit # tests that may be defined in .lua files. -# The "protocol" is that the .lua file must check for -# `KUMOMTA_RUN_UNIT_TESTS` being set in the environment -# to decide to run the tests, then use os.exit to terminate -# once it is done. -# -# ---- -# if os.getenv 'KUMOMTA_RUN_UNIT_TESTS' then -# kumo.configure_accounting_db_path(os.tmpname()) -# mod:test() -# os.exit(0) -# end -# ---- -LUA_FILE="${1}" -CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-$(pwd)/target} -KUMOMTA_RUN_UNIT_TESTS=1 ${CARGO_TARGET_DIR}/debug/kumod \ - --user `id -un` \ - --policy "${LUA_FILE}" +# The "protocol" is that the module defines a mod:test() method +# that will be called to perform regression tests +CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-${PWD}/target} + +for candidate in /opt/kumomta/sbin/kumod "${CARGO_TARGET_DIR}/release/kumod" "${CARGO_TARGET_DIR}/debug/kumod" ; do + if test -x "${candidate}" ; then + KUMOD="${candidate}" + break; + fi +done + +if ! test -x "${KUMOD}" ; then + echo "Couldn't find kumod" + exit 1 +fi + +script=$(mktemp) +trap "rm -f -- '$script'" EXIT +cat >${script} <<-EOT +local kumo = require 'kumo' +package.path = 'assets/?.lua;' .. package.path + +kumo.configure_accounting_db_path(os.tmpname()) + +local function test_module(name) + local mod = require(name) + mod:test() +end + +test_module 'policy-extras.typing' +test_module 'policy-extras.queue' +test_module 'policy-extras.listener_domains' + +os.exit(0) +EOT + +${KUMOD} --policy $script