mirror/kumomta
1
0
Fork 0
mirror of https://github.com/mailscope/kumomta.git synced 2026-09-11 13:02:14 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
3174bd5a39fbca7e611069d2ebd4fd55af955e47
kumomta/docs/reference/kumod-metrics.json
T
Wez Furlong 423abc4704 add per-dispatcher state tracking and watchdog 
The ready queue maintainer now tracks for each dispatcher whether it
is making progress.  If it exceeds dispatcher_progress_watchdog_timeout
then the task will be aborted and any message(s) it held onto will
be returned to the scheduled queue.  This will free up the connection
slot and allow another connection to be attempted, potentially
making further progress.

refs: https://github.com/KumoCorp/kumomta/issues/539
2026-06-23 09:56:45 +01:00

1542 lines
52 KiB
JSON
Raw Blame History

[
{
"name": "bind_failures",
"help": "How many times that directly binding a source address has failed.",
"doc": "This generally indicates a configuration error where a source\nis trying to assign an IP address that is not plumbing on\nthe system on which kumod is running.",
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "bounce_classify_latency",
"help": "latency of bounce classification",
"doc": null,
"metric_type": "Histogram",
"label_names": [],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "connection_count",
"help": "The number of active outgoing connections in the system, keyed by the service name.",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"service"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "connection_count_by_provider",
"help": "number of active connections",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"provider"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "connection_count_by_provider_and_pool",
"help": "number of active connections",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"provider",
"pool"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "delayed_due_to_message_rate_throttle",
"help": "Number of times a message was delayed due to max_message_rate",
"doc": "Delayed in this context means that we moved the message back to its corresponding\nscheduled queue with a short retry time, as well as logging a `Delayed` log\nrecord.\n\nSustained increases in this value may indicate that the configured\nthrottles are too severe for your workload, but it is difficult to make\na definitive and generalized statement in these docs without understanding your\nworkload, policy and the purpose of those throttles.\n\nThe metric is tracked per `queue` label. The `queue` is the scheduled\nqueue name as described in [Queues](../../queues.md).",
"metric_type": "Counter",
"label_names": [
"queue"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "delayed_due_to_ready_queue_full",
"help": "Number of times a message was delayed due to the corresponding ready queue being full.",
"doc": "Delayed in this context means that we moved the message back to its corresponding\nscheduled queue with a short retry time, as well as logging a `Delayed` log\nrecord.\n\nTransient spikes in this value indicate normal operation and that the system\nis keeping things within your memory budget.\n\nHowever, sustained increases in this value may indicate that the\n[max_ready](../../kumo/make_egress_path/max_ready.md)\nconfiguration for the associated egress path is under-sized for your workload,\nand that you should carefully consider the information in\n[Budgeting/Tuning Memory](../../memory.md#budgetingtuning-memory)\nto decide whether increasing `max_ready` is appropriate, otherwise you risk\npotentially over-provisioning the system.\n\nThe metric is tracked per `queue` label. The `queue` is the scheduled\nqueue name as described in [Queues](../../queues.md).\n\nSee [ready_full](ready_full.md) for the equivalent metric tracked\nby the ready queue name, which can be helpful to understand which\negress path configuration you might want to examine.",
"metric_type": "Counter",
"label_names": [
"queue"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "delayed_due_to_throttle_insert_ready",
"help": "number of times a message was delayed due to the throttle_insert_ready_queue event",
"doc": "Delayed in this context means that we moved the message back to its corresponding\nscheduled queue with a short retry time, as well as logging a `Delayed` log\nrecord.\n\nThe [throttle_insert_ready_queue](../../events/throttle_insert_ready_queue.md)\nevent is implemented either directly in your policy, or indirectly via policy\nhelpers, such as the queues helper when configured to throttle campaigns\nor tenants.\n\nSustained increases in this value may indicate that the configured\nthrottles are too severe for your workload, but it is difficult to make\na definitive and generalized statement in these docs without understanding your\nworkload, policy and the purpose of those throttles.\n\nThe metric is tracked per `queue` label. The `queue` is the scheduled\nqueue name as described in [Queues](../../queues.md).",
"metric_type": "Counter",
"label_names": [
"queue"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "deliver_message_latency_rollup",
"help": "how many seconds a deliver_message call takes for a given protocol",
"doc": null,
"metric_type": "Histogram",
"label_names": [
"service"
],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "disk_free_bytes",
"help": "number of available bytes in a monitored location",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "disk_free_inodes",
"help": "number of available inodes in a monitored location",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "disk_free_inodes_percent",
"help": "percentage of available inodes in a monitored location",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "disk_free_percent",
"help": "percentage of available bytes in a monitored location",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "dispatcher_watchdog_aborted_total",
"help": "total number of dispatcher tasks aborted by the progress watchdog",
"doc": "{{since('dev')}}",
"metric_type": "Counter",
"label_names": [
"service"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "dkim_signer_cache_hit",
"help": "How many dkim signer requests hit cache.",
"doc": "This is redundant with the newer\n[lruttl_hit_count{cache_name=\"dkim_signer_cache\"}](lruttl_hit_count.md)\ncounter.",
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "dkim_signer_cache_lookup_count",
"help": "How many cache dkim signer requests occurred.",
"doc": "This is redundant with the newer\n[lruttl_lookup_count{cache_name=\"dkim_signer_cache\"}](lruttl_lookup_count.md)\ncounter.",
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "dkim_signer_cache_miss",
"help": "How many dkim signer requests miss cache.",
"doc": "This is redundant with the newer\n[lruttl_miss_count{cache_name=\"dkim_signer_cache\"}](lruttl_miss_count.md)\ncounter.",
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "dkim_signer_creation",
"help": "How many seconds it takes to create a signer on a cache miss.",
"doc": null,
"metric_type": "Histogram",
"label_names": [],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "dkim_signer_key_cache_hit",
"help": "How many cache dkim signer requests hit key cache.",
"doc": "This is redundant with the newer\n[lruttl_hit_count{cache_name=\"dkim_key_cache\"}](lruttl_hit_count.md)\ncounter.",
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "dkim_signer_key_cache_lookup_count",
"help": "How many cache dkim key requests occurred.",
"doc": "This is redundant with the newer\n[lruttl_lookup_count{cache_name=\"dkim_key_cache\"}](lruttl_lookup_count.md)\ncounter.",
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "dkim_signer_key_cache_miss",
"help": "How many cache dkim signer requests miss key cache.",
"doc": "This is redundant with the newer\n[lruttl_miss_count{cache_name=\"dkim_key_cache\"}](lruttl_miss_count.md)\ncounter.",
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "dkim_signer_key_fetch",
"help": "How long it takes to obtain a dkim key.",
"doc": "This measures that time that it takes to load dkim\nprivate keys from whatever storage medium is configured.",
"metric_type": "Histogram",
"label_names": [],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "dkim_signer_message_parse",
"help": "How many seconds it takes to parse messages as prep for signing.",
"doc": "Long durations may simply indicate that you have very large\nmessages passing through the system.",
"metric_type": "Histogram",
"label_names": [],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "dkim_signer_sign",
"help": "How many seconds it takes to dkim sign parsed messages.",
"doc": "Long durations may indicate that the system is over-provisioned\nand has insufficient CPU. You should check whether and how you\nmight have configured\n[kumo.dkim.set_signing_threads](../../kumo.dkim/set_signing_threads.md).",
"metric_type": "Histogram",
"label_names": [],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "dns_mx_resolve_cache_hit",
"help": "Total number of MailExchanger::resolve calls satisfied by level 1 cache.",
"doc": "Redundant with the newer [lruttl_hit_count{cache_name=\"dns_resolver_mx\"}](lruttl_hit_count.md)\nmetric.",
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "dns_mx_resolve_cache_miss",
"help": "Total number of MailExchanger::resolve calls that resulted in an MX DNS request to the next level of cache",
"doc": "Redundant with the newer [lruttl_miss_count{cache_name=\"dns_resolver_mx\"}](lruttl_miss_count.md)\nmetric.",
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "dns_mx_resolve_in_progress",
"help": "number of `MailExchanger::resolve` calls currently in progress.",
"doc": null,
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "dns_mx_resolve_status_fail",
"help": "Total number of failed `MailExchanger::resolve` calls.",
"doc": "Spikes may indicate an issue with your DNS configuration\nor infrastructure, or may simply indicate that the traffic\nis destined for bogus addresses.",
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "dns_mx_resolve_status_ok",
"help": "Total number of successful `MailExchanger::resolve` calls",
"doc": null,
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "egress_source_connection_failures_total",
"help": "Counts connection failures classified as belonging to one of the source-health failure classes. Increments regardless of whether `suspend_when_*` is configured on the source, so an operator can observe the underlying signal before opting in to auto-suspension.",
"doc": "{{since('dev')}}\n\nLabels:\n* `source` is the operator-defined egress source name (the\n `name` field of `kumo.make_egress_source`).\n* `kind` is one of:\n * `Unplumbed` — the source's local `source_address`\n returned `EADDRNOTAVAIL` on `bind()`. The IP address\n is not currently plumbed on this host.\n * `ProxyUnhealthy` — the configured proxy server was\n unreachable, timed out, or reported a bind failure\n for the requested source address.",
"metric_type": "Counter",
"label_names": [
"source",
"kind"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "egress_source_health_suspended",
"help": "`1` while an egress source is currently auto-suspended, `0` otherwise. Pool selection skips a source whose gauge is `1`, rolling the remaining suspension duration into the per-pool `min_delay`.",
"doc": "{{since('dev')}}\n\nLabels:\n* `source` is the operator-defined egress source name (the\n `name` field of `kumo.make_egress_source`).\n* `reason` indicates which rule's firing produced the\n current suspension:\n * `Unplumbed` — a `suspend_when_unplumbed` rule fired\n because the source's local `source_address` was not\n plumbed on this host. Plumb the address (or correct\n the configured `source_address`) to resolve.\n * `ProxyUnhealthy` — a `suspend_when_proxy_unhealthy`\n rule fired because the configured proxy server was\n unreachable or rejected the requested source address.\n Investigate the proxy service.",
"metric_type": "Gauge",
"label_names": [
"source",
"reason"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "egress_source_health_suspensions_total",
"help": "Increments each time a source transitions into the auto-suspended state due to one of its `suspend_when_*` rules firing. Re-triggering the same rule while the suspension is already active does not increment this counter; it only counts state transitions.",
"doc": "{{since('dev')}}\n\nLabels:\n* `source` is the operator-defined egress source name (the\n `name` field of `kumo.make_egress_source`).\n* `reason` indicates which rule fired:\n * `Unplumbed` — a `suspend_when_unplumbed` rule fired\n because the source's local `source_address` was not\n plumbed on this host. Plumb the address (or correct\n the configured `source_address`) to resolve.\n * `ProxyUnhealthy` — a `suspend_when_proxy_unhealthy`\n rule fired because the configured proxy server was\n unreachable or rejected the requested source address.\n Investigate the proxy service.",
"metric_type": "Counter",
"label_names": [
"source",
"reason"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "log_hook_backlog_count",
"help": "how many times processing of a log event hit the back_pressure in a hook",
"doc": null,
"metric_type": "Counter",
"label_names": [
"logger"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "log_submit_full",
"help": "how many times submission of a log event hit the back_pressure",
"doc": null,
"metric_type": "Counter",
"label_names": [
"logger"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "log_submit_latency",
"help": "latency of log event submission operations",
"doc": null,
"metric_type": "Histogram",
"label_names": [
"logger"
],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "lruttl_cache_size",
"help": "The number of items currently contained in an lruttl cache.",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Gauge",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_error_count",
"help": "how many times a lruttl cache population resulted in an error",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_evict_count",
"help": "how many times a lruttl cache evicted an item due to capacity constraints",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_expire_count",
"help": "how many times a lruttl cache removed an item due to ttl expiration",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_hit_count",
"help": "how many times a lruttl cache lookup was a hit for a given cache",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_insert_count",
"help": "how many times a lruttl cache was populated via unconditional insert",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_lookup_count",
"help": "How many times a lruttl cache lookup was initiated for a given cache.",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_miss_count",
"help": "how many times a lruttl cache lookup was a miss for a given cache",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_populated_count",
"help": "how many times a lruttl cache lookup resulted in performing the work to populate the entry",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_stale_count",
"help": "how many times a lruttl cache population was satisfied by a stale value",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_waiting_populate",
"help": "how many tasks are currently waiting for a cache entry to populate",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Gauge",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lua_count",
"help": "the number of lua contexts currently alive",
"doc": null,
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lua_event_latency",
"help": "how long a given lua event callback took",
"doc": null,
"metric_type": "Histogram",
"label_names": [
"event"
],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "lua_event_started",
"help": "Incremented each time we start to call a lua event callback. Use lua_event_latency_count to track completed events",
"doc": null,
"metric_type": "Counter",
"label_names": [
"event"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lua_load_count",
"help": "how many times the policy lua script has been loaded into a new context",
"doc": null,
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lua_spare_count",
"help": "the number of lua contexts available for reuse in the pool",
"doc": null,
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "memoize_cache_hit_count",
"help": "How many times a memoize cache lookup was a hit for a given cache.",
"doc": "Redundant with the newer [lruttl_hit_count](lruttl_hit_count.md) metric.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "memoize_cache_lookup_count",
"help": "How many times a memoize cache lookup was initiated for a given cache.",
"doc": "Redundant with the newer [lruttl_lookup_count](lruttl_lookup_count.md) metric.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "memoize_cache_miss_count",
"help": "How many times a memoize cache lookup was a miss for a given cache",
"doc": "Redundant with the newer [lruttl_miss_count](lruttl_miss_count.md) metric.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "memoize_cache_populated_count",
"help": "How many times a memoize cache lookup resulted in performing the work to populate the entry",
"doc": "Redundant with the newer [lruttl_populated_count](lruttl_populated_count.md) metric.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "memory_limit",
"help": "soft memory limit measured in bytes",
"doc": null,
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "memory_low_count",
"help": "How many times the low memory threshold was exceeded.",
"doc": null,
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "memory_low_thresh",
"help": "low memory threshold measured in bytes",
"doc": null,
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "memory_over_limit_count",
"help": "how many times the soft memory limit was exceeded",
"doc": null,
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "memory_usage",
"help": "number of bytes of used memory (Resident Set Size)",
"doc": null,
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "memory_usage_rust",
"help": "number of bytes of used memory (allocated by Rust)",
"doc": null,
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "message_count",
"help": "Total number of Message objects.",
"doc": "This encompasses all Message objects in various states, whether\nthey are in a queue, moving between queues, being built as part\nof an injection, pending logging, message metadata and/or data\nmay be either resident or offloaded to spool.",
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "message_data_load_latency",
"help": "How many seconds it takes to load message data from spool.",
"doc": "High values indicate IO pressure which may be caused\nby policy that operates on the message body post-reception.\nWe recommend *avoiding* logging header values as that is\nthe most common cause of this metric spiking and has\nthe biggest impact in resolving it.\n\nIO pressure may also be alleviated by tuning other constraints and/or\n[RocksDB Parameters](../../kumo/define_spool/rocks_params.md)",
"metric_type": "Histogram",
"label_names": [],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "message_data_resident_count",
"help": "Total number of Message objects with body data loaded.",
"doc": "Tracks how many messages have their `data` resident\nin memory. This may be because they have not yet saved\nit, or because the message is being processed and the\ndata is either required to be in memory in order to\ndeliver the message, or because logging or other\npost-injection policy is configured to operate on\nthe message.",
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "message_meta_load_latency",
"help": "How long it takes to load message metadata from spool",
"doc": "High values indicate IO pressure which may be\nalleviated by tuning other constraints and/or\n[RocksDB Parameters](../../kumo/define_spool/rocks_params.md)",
"metric_type": "Histogram",
"label_names": [],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "message_meta_resident_count",
"help": "Total number of Message objects with metadata loaded.",
"doc": "Tracks how many messages have their `meta` data resident\nin memory. This may be because they have not yet saved\nit, or because the message is being processed and the\nmetadata is required for that processing.",
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "message_save_latency",
"help": "How many seconds it takes to save a message to spool.",
"doc": "This metric encompasses the elapsed time to saved\neither or both the `meta` and `data` portions of a\nmessage to spool.\n\nHigh values indicate IO pressure which may be\nalleviated by tuning other constraints and/or\n[RocksDB Parameters](../../kumo/define_spool/rocks_params.md)",
"metric_type": "Histogram",
"label_names": [],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "process_cpu_usage_normalized",
"help": "The sum of the process CPU usage for each CPU in the system, divided by the number of CPUs.",
"doc": "100% in this metric indicates that all CPU cores are 100% busy.\n\nThis metric is scoped to the service process, reflecting the CPU used only\nby the process and not the system as a whole.",
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "process_cpu_usage_sum",
"help": "The sum of the process CPU usage for each CPU in the system, can add up to more than 100%.",
"doc": "Each CPU has a value from 0-100% busy; a value of 100% in this metric\nindicates that the load is equivalent to one fully utilized CPU.\n\nA multi-CPU system can report more than 100% in this metric; a dual-CPU\nsystem reporting 200% indicates that both CPUs are fully utilized.\n\nSee process_cpu_usage_normalized for a version of this metric that scales from\n0% (totally idle) to 100% (totally saturated).\n\nThis metric is scoped to the service process, reflecting the CPU used only\nby the process and not the system as a whole.",
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "proxy_connection_failures",
"help": "How many times a connection attempt to a proxy server has failed.",
"doc": "This might indicate either a configuration error (eg: an\nincorrect proxy server has been configured) or a\nservice interruption with that proxy server.",
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "queue_insert_latency",
"help": "latency of QueueManager::insert operations",
"doc": null,
"metric_type": "Histogram",
"label_names": [],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "queue_resolve_latency",
"help": "latency of QueueManager::resolve operations",
"doc": null,
"metric_type": "Histogram",
"label_names": [],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "queued_count_by_provider",
"help": "number of messages in the scheduled and ready queue",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"provider"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "queued_count_by_provider_and_pool",
"help": "number of messages in the scheduled and ready queue",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"provider",
"pool"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "ready_count",
"help": "number of messages in the ready queue",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"service"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "ready_full",
"help": "number of times a message could not fit in the ready queue.",
"doc": "See [delayed_due_to_ready_queue_full](delayed_due_to_ready_queue_full.md)\nfor the equivalent metric tracked by scheduled queue name, as well as\na discussion on what this event means.",
"metric_type": "Counter",
"label_names": [
"service"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "ready_queue_insert_latency",
"help": "latency of ReadyQueue::insert operations",
"doc": null,
"metric_type": "Histogram",
"label_names": [],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "redis_operation_latency",
"help": "The latency of an operation talking to Redis.",
"doc": "{{since('2026.03.04-bb93ecb1')}}\n\nThe `service` key represents the redis server/service. It is not\na direct match to a server name as it is really a hash of the\noverall redis configuration information used in the client.\nIt might look something like:\n`redis://127.0.0.1:24419,redis://127.0.0.1:7779,redis://127.0.0.1:29469-2ce79dd1`\nfor a cluster configuration, or `redis://127.0.0.1:16267-f4da6e64`\nfor a single node cluster configuration.\nYou should anticipate that the `-HEX` suffix can and will change\nin an unspecified way as you vary the redis connection parameters.\n\nThe `operation` key indicates the operation, which can be a `ping`,\na `query` or a `script`.\n\n`status` will be either `ok` or `error` to indicate whether this\nis tracking a successful or failed operation.\n\nSince histograms track a count of operations, you can track the\nrate of `redis_operation_latency_count` where `status=error`\nto have an indication of the failure rate of redis operations.",
"metric_type": "Histogram",
"label_names": [
"service",
"operation",
"status"
],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "rocks_spool_actual_delayed_write_rate",
"help": "Current delayed write rate (bytes/second) applied by rocksdb to throttle foreground writers. 0 means no slowdown is in effect.",
"doc": "{{since('dev')}}\n\nA non-zero value means rocksdb is intentionally slowing writers\ndown because compaction or flush is falling behind. This is the\nearly-warning signal that precedes a full write stop: if this\nremains non-zero for an extended period, investigate the\ncompaction backlog\n(`rocks_spool_estimate_pending_compaction_bytes`) and underlying\ndisk throughput before the database transitions to\n`rocks_spool_write_stopped == 1`.",
"metric_type": "Gauge",
"label_names": [
"path"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "rocks_spool_background_errors",
"help": "Accumulated count of background errors encountered by the rocksdb instance (failed flushes or compactions, typically caused by I/O errors such as missing or corrupt SST files, ENOSPC, or permission problems).",
"doc": "{{since('dev')}}\n\nThis counter is **monotonic** for the lifetime of the process: it\ndoes not decrease when rocksdb auto-resumes from transient errors\nsuch as a brief ENOSPC. A non-zero value therefore does not\nnecessarily mean the database is currently wedged; it means at\nleast one background error has occurred since the process started.\n\nFor SRE monitoring, alert on the **rate of change** (e.g.\n`increase(rocks_spool_background_errors[5m]) > 0`) to catch new\noccurrences. For the actionable \"the database is wedged right\nnow and we are shedding load\" signal, page on\n`rocks_spool_load_shed_active` instead, which combines this\ncounter, foreground read/write errors, and rocksdb error\nseverity into a single latched indicator.",
"metric_type": "Gauge",
"label_names": [
"path"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "rocks_spool_cache_total",
"help": "Approximate memory (bytes) usage by cache.",
"doc": "This may be useful when understanding the memory usage of\nthe system.",
"metric_type": "Gauge",
"label_names": [
"path"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "rocks_spool_compaction_pending",
"help": "Set to 1 when at least one compaction is pending for this rocksdb instance, 0 otherwise.",
"doc": "{{since('dev')}}\n\nBrief flapping is normal under write load. A value of 1 that\npersists alongside `rocks_spool_num_running_compactions == 0` is\nsuspicious and suggests the compaction worker is not making\nprogress.",
"metric_type": "Gauge",
"label_names": [
"path"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "rocks_spool_estimate_pending_compaction_bytes",
"help": "Estimated total bytes that compaction needs to rewrite to bring all levels back under their target sizes.",
"doc": "{{since('dev')}}\n\nThis is a backlog indicator. Steady-state values depend heavily\non write rate, compression, and the configured compaction style,\nso absolute thresholds should be derived from each deployment's\nbaseline. Unbounded growth over a multi-hour window indicates\nthat compaction cannot keep up with the write rate, which\neventually leads to write slowdown\n(`rocks_spool_actual_delayed_write_rate` becomes non-zero) and\nthen to write stop (`rocks_spool_write_stopped` becomes 1).\n\nOnly meaningful for level-style compaction.",
"metric_type": "Gauge",
"label_names": [
"path"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "rocks_spool_load_shed_active",
"help": "Set to 1 when this spool's load-shedding gate is latched, 0 otherwise. When set, ingress paths (SMTP, HTTP inject) reject traffic and foreground store/remove operations fail fast rather than stall.",
"doc": "{{since('dev')}}\n\nThe gate latches in either of two ways:\n\n* **Immediate**: a foreground spool operation (load, store,\n remove) returns a rocksdb error classified as definitively\n bad (`Corruption` or `IOError` -- e.g. a missing or corrupt\n SST file discovered during a read). These conditions have\n no transient interpretation, so the gate latches on the\n first such observation.\n* **Debounced**: less specific failure signals --\n `background-errors` has grown since this process started, or\n foreground operations have returned non-fatal errors --\n sustained continuously for the configured\n `error_latch_duration` (default 15s). This filters out\n brief auto-resumed errors.\n\nIf `allow_error_unlatch` is enabled (the default), the gate\nauto-clears after `error_unlatch_duration` of observed recovery\n(default 5 minutes) with no new errors of either class.\nOtherwise it stays set until the process is restarted.\n\nSREs should treat any sustained non-zero value as an\noperator-actionable incident; pair this metric with\n`rocks_spool_background_errors` to understand why.",
"metric_type": "Gauge",
"label_names": [
"path"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "rocks_spool_mem_table_readers_total",
"help": "Approximate memory usage (bytes) of all the table readers.",
"doc": "This may be useful when understanding the memory usage of\nthe system.",
"metric_type": "Gauge",
"label_names": [
"path"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "rocks_spool_mem_table_total",
"help": "Approximate memory usage (bytes) of all the mem-tables.",
"doc": "This may be useful when understanding the memory usage of\nthe system.",
"metric_type": "Gauge",
"label_names": [
"path"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "rocks_spool_mem_table_unflushed",
"help": "Approximate memory usage (bytes) of un-flushed mem-tables.",
"doc": "This may be useful when understanding the memory usage of\nthe system.",
"metric_type": "Gauge",
"label_names": [
"path"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "rocks_spool_num_running_compactions",
"help": "Number of background compactions currently running for this rocksdb instance.",
"doc": "{{since('dev')}}\n\nIn a healthy, actively-written spool this is typically non-zero\nin bursts. A value persistently stuck at 0 while\n`rocks_spool_compaction_pending` or\n`rocks_spool_estimate_pending_compaction_bytes` is growing is a\nstrong indicator that the background worker is wedged --\ncross-reference `rocks_spool_write_stopped` and\n`rocks_spool_background_errors`.",
"metric_type": "Gauge",
"label_names": [
"path"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "rocks_spool_write_stopped",
"help": "Set to 1 when the rocksdb instance is currently refusing writes at the WriteController layer (memtable count or L0 file count reached the stop threshold), 0 otherwise.",
"doc": "{{since('dev')}}\n\nThis reflects rocksdb's own `is-write-stopped` property and\nindicates backpressure rather than a fatal background error.\nHealthy databases under bursty load may briefly report 1 here.\nFor the \"the database is wedged due to a background error\"\nsignal, see `rocks_spool_load_shed_active` instead.",
"metric_type": "Gauge",
"label_names": [
"path"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "scheduled_by_domain",
"help": "number of messages in the scheduled queue for a specific domain",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"domain"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "scheduled_by_tenant",
"help": "number of messages in the scheduled queue for a specific tenant",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"tenant"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "scheduled_by_tenant_campaign",
"help": "number of messages in the scheduled queue for a specific tenant and campaign combination",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"tenant",
"campaign"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "scheduled_count",
"help": "number of messages in the scheduled queue.",
"doc": "The metric is tracked per `queue` label. The `queue` is the scheduled\nqueue name as described in [Queues](../../queues.md).",
"metric_type": "Gauge",
"label_names": [
"queue"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "scheduled_count_total",
"help": "total number of messages across all scheduled queues.",
"doc": "This counter sums up the number of messages currently sitting in all scheduled queues.",
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "scheduled_queue_count",
"help": "how many scheduled queues are tracked by the QueueManager",
"doc": null,
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "scheduled_queue_maintainer_count",
"help": "How many scheduled queues have active maintainer tasks.",
"doc": null,
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "smtp_server_rejections",
"help": "number of Rejection records logged by the smtp server",
"doc": null,
"metric_type": "Counter",
"label_names": [
"service"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "smtpsrv_process_data_duration",
"help": "how long it takes to process the DATA portion and enqueue",
"doc": null,
"metric_type": "Histogram",
"label_names": [],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "smtpsrv_read_data_duration",
"help": "how long it takes to receive the DATA portion",
"doc": null,
"metric_type": "Histogram",
"label_names": [],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "smtpsrv_transaction_duration",
"help": "how long an incoming SMTP transaction takes",
"doc": null,
"metric_type": "Histogram",
"label_names": [],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "system_cpu_usage_normalized",
"help": "The sum of the system-wide CPU usage for each CPU in the system, divided by the number of CPUs.",
"doc": "100% in this metric indicates that all CPU cores are 100% busy.\n\nThis metric is scoped to the system, reflecting the total load on the\nsystem, not just from the kumo related process(es).",
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "system_cpu_usage_sum",
"help": "The sum of the system-wide CPU usage for each CPU in the system, can add up to more than 100%.",
"doc": "Each CPU has a value from 0-100% busy; a value of 100% in this metric\nindicates that the load is equivalent to one fully utilized CPU.\n\nA multi-CPU system can report more than 100% in this metric; a dual-CPU\nsystem reporting 200% indicates that both CPUs are fully utilized.\n\nSee system_cpu_usage_normalized for a version of this metric that scales from\n0% (totally idle) to 100% (totally saturated).\n\nThis metric is scoped to the system, reflecting the total load on the\nsystem, not just from the kumo related process(es).",
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "thread_pool_parked",
"help": "number of parked (idle) threads in a thread pool",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"pool"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "thread_pool_size",
"help": "number of threads in a thread pool",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"pool"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "timeq_pop_interval",
"help": "The amount of time that passes between calls to `TimeQ::pop`.",
"doc": "This metric is not generally interesting and does not typically\nneed to be charted in a dashboard.",
"metric_type": "Histogram",
"label_names": [],
"buckets": [
3.0,
4.0,
5.0,
8.0,
10.0,
12.0,
15.0,
20.0,
25.0,
30.0
],
"pruning": "NonPruning"
},
{
"name": "timeq_pop_latency",
"help": "The number of seconds that passes between calls to a singleon timerwheel pop.",
"doc": "This gives an indication of how heavily loaded the timerwheel buckets might be, but is not\ngenerally useful to chart.",
"metric_type": "Histogram",
"label_names": [],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "timeq_pop_tardiness",
"help": "The time difference between the due and current time for a singleon timerwheel pop.",
"doc": "This gives an indication of whether the scheduled queue\nmaintainer is keeping up with the load. It is generally\nacceptable for this value to be a few seconds \"late\" due\nto a combination of time wheel bucket granularity and\noverall scheduling priority.",
"metric_type": "Histogram",
"label_names": [],
"buckets": [
0.25,
0.5,
1.0,
2.5,
3.0,
5.0,
10.0,
15.0
],
"pruning": "NonPruning"
},
{
"name": "timeq_reinsert_tardiness",
"help": "The time difference between the due and current time for a singleon timerwheel reinsertion.",
"doc": "This gives an indication of whether the scheduled queue\nmaintainer is keeping up with the load. It is generally\nacceptable for this value to be a few seconds \"late\" due\nto a combination of time wheel bucket granularity and\noverall scheduling priority.",
"metric_type": "Histogram",
"label_names": [],
"buckets": [
0.25,
0.5,
1.0,
2.5,
3.0,
5.0,
10.0,
15.0,
30.0,
45.0,
60.0,
90.0,
180.0,
360.0,
720.0
],
"pruning": "NonPruning"
},
{
"name": "total_connection_count",
"help": "total number of active connections ever made",
"doc": null,
"metric_type": "Counter",
"label_names": [
"service"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "total_connections_denied",
"help": "total number of connections rejected due to load shedding or concurrency limits",
"doc": null,
"metric_type": "Counter",
"label_names": [
"service"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "total_messages_delivered",
"help": "total number of messages ever delivered",
"doc": null,
"metric_type": "Counter",
"label_names": [
"service"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "total_messages_delivered_by_provider",
"help": "total number of messages ever delivered",
"doc": null,
"metric_type": "Counter",
"label_names": [
"provider"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "total_messages_delivered_by_provider_and_source",
"help": "total number of messages ever delivered",
"doc": null,
"metric_type": "Counter",
"label_names": [
"provider",
"source",
"pool"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "total_messages_fail",
"help": "total number of message delivery attempts that permanently failed",
"doc": null,
"metric_type": "Counter",
"label_names": [
"service"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "total_messages_fail_by_provider",
"help": "total number of message delivery attempts that permanently failed",
"doc": null,
"metric_type": "Counter",
"label_names": [
"provider"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "total_messages_fail_by_provider_and_source",
"help": "total number of message delivery attempts that permanently failed",
"doc": null,
"metric_type": "Counter",
"label_names": [
"provider",
"source",
"pool"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "total_messages_received",
"help": "total number of messages ever received",
"doc": null,
"metric_type": "Counter",
"label_names": [
"service"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "total_messages_transfail",
"help": "total number of message delivery attempts that transiently failed",
"doc": null,
"metric_type": "Counter",
"label_names": [
"service"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "total_messages_transfail_by_provider",
"help": "total number of message delivery attempts that transiently failed",
"doc": null,
"metric_type": "Counter",
"label_names": [
"provider"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "total_messages_transfail_by_provider_and_source",
"help": "total number of message delivery attempts that transiently failed",
"doc": null,
"metric_type": "Counter",
"label_names": [
"provider",
"source",
"pool"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "total_qmaint_runs",
"help": "Total number of times a scheduled queue maintainer was run.",
"doc": "This metric is not generally useful to chart.\nIt gives an indication that schedule queue maintainers are\nticking over, but it is difficult to reason much beyond that\nthat is happening.",
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "total_readyq_runs",
"help": "total number of times a readyq maintainer was run",
"doc": null,
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "user_lua_latency",
"help": "How many seconds something user-defined took to run in your lua policy.",
"doc": "This histogram is updated by policy scripts that employ the\n[kumo.time.start_timer](../../kumo.time/start_timer.md) function\nto record a duration in the policy.\n\nThe `label` is whatever you specified as the label(s) to the various\n`kumo.time.start_timer` calls in the policy.",
"metric_type": "Histogram",
"label_names": [
"label"
],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
}
]
Reference in New Issue View Git Blame Copy Permalink