Files
kumomta/docs/reference/proxy-server-metrics.json
T
Wez Furlong 4170955400 mailexchanger: Distinguish MX lookup timeout causes in logs and metrics
Lets operators tell local concurrency permit wait timeouts apart from real
remote resolver query timeouts, to avoid misdiagnosing overload as remote
deliverability failure.

Adds metrics dns_mx_resolve_permit_timeout, dns_mx_resolve_query_timeout
and dns_mx_resolve_permit_waiters.
2026-10-08 14:58:52 +01:00

661 lines
23 KiB
JSON

[
{
"name": "disk_free_bytes",
"help": "number of available bytes in a monitored location",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "disk_free_inodes",
"help": "number of available inodes in a monitored location",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "disk_free_inodes_percent",
"help": "percentage of available inodes in a monitored location",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "disk_free_percent",
"help": "percentage of available bytes in a monitored location",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "dns_mx_resolve_cache_hit",
"help": "Total number of MailExchanger::resolve calls satisfied by level 1 cache.",
"doc": "Redundant with the newer [lruttl_hit_count{cache_name=\"dns_resolver_mx\"}](lruttl_hit_count.md)\nmetric.",
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "dns_mx_resolve_cache_miss",
"help": "Total number of MailExchanger::resolve calls that resulted in an MX DNS request to the next level of cache",
"doc": "Redundant with the newer [lruttl_miss_count{cache_name=\"dns_resolver_mx\"}](lruttl_miss_count.md)\nmetric.",
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "dns_mx_resolve_in_progress",
"help": "number of `MailExchanger::resolve` calls currently in progress.",
"doc": null,
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "dns_mx_resolve_mta_sts_impossible",
"help": "Total number of MailExchanger::resolve calls that failed because the\ndomain published an MTA-STS enforce policy that permits none of its own\nMX hosts. Such domains are undeliverable until they fix their policy.",
"doc": null,
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "dns_mx_resolve_permit_timeout",
"help": "Total number of MX lookups that timed out while waiting for an MX\nconcurrency permit before any query could be sent. Counts local\nconcurrency pressure, not any response from the destination.",
"doc": null,
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "dns_mx_resolve_permit_waiters",
"help": "Number of MX lookups currently blocked waiting for an MX concurrency\npermit before their query can be sent.",
"doc": null,
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "dns_mx_resolve_query_timeout",
"help": "Total number of MX lookups that timed out after the query was sent, while\nwaiting for the resolver to answer.",
"doc": null,
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "dns_mx_resolve_status_fail",
"help": "Total number of failed `MailExchanger::resolve` calls.",
"doc": "Spikes may indicate an issue with your DNS configuration\nor infrastructure, or may simply indicate that the traffic\nis destined for bogus addresses.",
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "dns_mx_resolve_status_ok",
"help": "Total number of successful `MailExchanger::resolve` calls",
"doc": null,
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "log_consumer_bytes_behind",
"help": "The compressed size, in bytes, of whole log segments awaiting processing by\na consumer.",
"doc": "This counts whole unread segments only, not the remainder of the segment\nthat the consumer is partway through processing.\n\nLabelled by:\n* `log_dir` - the log directory being scanned\n* `checkpoint_name` - the name set by the tailer of the consumer for its\n checkpoint.",
"metric_type": "Gauge",
"label_names": [
"log_dir",
"checkpoint_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "log_consumer_lag_seconds",
"help": "Age in seconds of the oldest log segment not yet started by a consumer,\ncounted from the modification time of that segment.",
"doc": "Such a segment does not exist once the consumer has reached the newest log\nsegment. The gauge reports 0 in that case.\n\nLabelled by:\n\n* `log_dir` - the log directory being scanned\n* `checkpoint_name` - the name set by the tailer of the consumer for its\n checkpoint.",
"metric_type": "Gauge",
"label_names": [
"log_dir",
"checkpoint_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "log_consumer_segments_behind",
"help": "The number of whole log segments not yet processed by a consumer.",
"doc": "Labelled by:\n* `log_dir` - the log directory being scanned\n* `checkpoint_name` - the name set by the tailer of the consumer for its\n checkpoint.\n\nA sample value is\n`log_consumer_segments_behind{log_dir=\"/var/log/kumomta\",checkpoint_name=\"cp\"} 3`.",
"metric_type": "Gauge",
"label_names": [
"log_dir",
"checkpoint_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_cache_size",
"help": "The number of items currently contained in an lruttl cache.",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Gauge",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_error_count",
"help": "how many times a lruttl cache population resulted in an error",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_evict_count",
"help": "how many times a lruttl cache evicted an item due to capacity constraints",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_expire_count",
"help": "how many times a lruttl cache removed an item due to ttl expiration",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_hit_count",
"help": "how many times a lruttl cache lookup was a hit for a given cache",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_insert_count",
"help": "how many times a lruttl cache was populated via unconditional insert",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_lookup_count",
"help": "How many times a lruttl cache lookup was initiated for a given cache.",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_miss_count",
"help": "how many times a lruttl cache lookup was a miss for a given cache",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_populated_count",
"help": "how many times a lruttl cache lookup resulted in performing the work to populate the entry",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_stale_count",
"help": "how many times a lruttl cache population was satisfied by a stale value",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lruttl_waiting_populate",
"help": "how many tasks are currently waiting for a cache entry to populate",
"doc": "The `cache_name` label identifies which cache. See [kumo.set_lruttl_cache_capacity](../../kumo/set_lruttl_cache_capacity.md) for a list of caches.",
"metric_type": "Gauge",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lua_count",
"help": "the number of lua contexts currently alive",
"doc": null,
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lua_event_latency",
"help": "how long a given lua event callback took",
"doc": null,
"metric_type": "Histogram",
"label_names": [
"event"
],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "lua_event_started",
"help": "Incremented each time we start to call a lua event callback. Use lua_event_latency_count to track completed events",
"doc": null,
"metric_type": "Counter",
"label_names": [
"event"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lua_load_count",
"help": "how many times the policy lua script has been loaded into a new context",
"doc": null,
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "lua_spare_count",
"help": "the number of lua contexts available for reuse in the pool",
"doc": null,
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "memoize_cache_hit_count",
"help": "How many times a memoize cache lookup was a hit for a given cache.",
"doc": "Redundant with the newer [lruttl_hit_count](lruttl_hit_count.md) metric.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "memoize_cache_lookup_count",
"help": "How many times a memoize cache lookup was initiated for a given cache.",
"doc": "Redundant with the newer [lruttl_lookup_count](lruttl_lookup_count.md) metric.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "memoize_cache_miss_count",
"help": "How many times a memoize cache lookup was a miss for a given cache",
"doc": "Redundant with the newer [lruttl_miss_count](lruttl_miss_count.md) metric.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "memoize_cache_populated_count",
"help": "How many times a memoize cache lookup resulted in performing the work to populate the entry",
"doc": "Redundant with the newer [lruttl_populated_count](lruttl_populated_count.md) metric.",
"metric_type": "Counter",
"label_names": [
"cache_name"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "memory_limit",
"help": "soft memory limit measured in bytes",
"doc": null,
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "memory_low_count",
"help": "How many times the low memory threshold was exceeded.",
"doc": null,
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "memory_low_thresh",
"help": "low memory threshold measured in bytes",
"doc": null,
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "memory_over_limit_count",
"help": "how many times the soft memory limit was exceeded",
"doc": null,
"metric_type": "Counter",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "memory_usage",
"help": "number of bytes of used memory that drives memory limit decisions.\nWhen sourced from a cgroup this is the working set\n(memory.current - inactive_file, floored by anonymous memory);\notherwise it is the Resident Set Size from /proc/self/statm.",
"doc": null,
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "memory_usage_rust",
"help": "number of bytes of used memory (allocated by Rust)",
"doc": null,
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "process_cpu_usage_normalized",
"help": "The sum of the process CPU usage for each CPU in the system, divided by the number of CPUs.",
"doc": "100% in this metric indicates that all CPU cores are 100% busy.\n\nThis metric is scoped to the service process, reflecting the CPU used only\nby the process and not the system as a whole.",
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "process_cpu_usage_sum",
"help": "The sum of the process CPU usage for each CPU in the system, can add up to more than 100%.",
"doc": "Each CPU has a value from 0-100% busy; a value of 100% in this metric\nindicates that the load is equivalent to one fully utilized CPU.\n\nA multi-CPU system can report more than 100% in this metric; a dual-CPU\nsystem reporting 200% indicates that both CPUs are fully utilized.\n\nSee process_cpu_usage_normalized for a version of this metric that scales from\n0% (totally idle) to 100% (totally saturated).\n\nThis metric is scoped to the service process, reflecting the CPU used only\nby the process and not the system as a whole.",
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "proxy_accept_errors_total",
"help": "Total number of accept() errors that caused the listener to pause before\nretrying, such as file-descriptor exhaustion.",
"doc": "{{since('dev')}}",
"metric_type": "Counter",
"label_names": [
"listener"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "proxy_active_connections",
"help": "Current number of active proxy connections.",
"doc": "This gauge shows the number of connections currently being proxied.\nIt increments when a connection is accepted and decrements when\nthe connection closes (successfully or with error).",
"metric_type": "Gauge",
"label_names": [
"listener"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "proxy_bytes_client_to_dest_total",
"help": "Total bytes transferred from client to destination.",
"doc": "This counter tracks the total number of bytes flowing from proxy clients\nto their intended destinations (upstream direction).",
"metric_type": "Counter",
"label_names": [
"listener"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "proxy_bytes_dest_to_client_total",
"help": "Total bytes transferred from destination to client.",
"doc": "This counter tracks the total number of bytes flowing from destinations\nback to proxy clients (downstream direction).",
"metric_type": "Counter",
"label_names": [
"listener"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "proxy_connections_accepted_total",
"help": "Total number of incoming connections accepted by the proxy.",
"doc": "This counter increments each time a new client connection is accepted\nby a proxy listener, before any SOCKS5 handshake begins.",
"metric_type": "Counter",
"label_names": [
"listener"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "proxy_connections_completed_total",
"help": "Total number of proxy sessions that completed successfully.",
"doc": "This counter increments when a proxy session completes without error,\nmeaning the client connected, was proxied to the destination, and\nboth sides closed cleanly.",
"metric_type": "Counter",
"label_names": [
"listener"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "proxy_connections_denied_total",
"help": "Total number of connections closed immediately for exceeding the\n`max_connections` limit of the listener.",
"doc": "{{since('dev')}}",
"metric_type": "Counter",
"label_names": [
"listener"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "proxy_connections_failed_total",
"help": "Total number of connections that failed during handshake or proxying.",
"doc": "This counter increments when a connection fails due to handshake errors,\nauthentication failures, timeouts, or I/O errors during proxying.",
"metric_type": "Counter",
"label_names": [
"listener"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "proxy_outbound_connections_total",
"help": "Total number of outbound connections made to destinations.",
"doc": "This counter tracks connections by destination IP address.\nNote: This can create high cardinality if your proxy connects to many\nunique destinations. The metric uses a pruning counter registry to\nmitigate memory impact.",
"metric_type": "Counter",
"label_names": [
"listener",
"destination"
],
"buckets": [],
"pruning": "Pruning"
},
{
"name": "proxy_tls_handshake_failures_total",
"help": "Total number of TLS handshake failures.",
"doc": "This counter increments when TLS is enabled on a listener and\nthe TLS handshake with a client fails.",
"metric_type": "Counter",
"label_names": [
"listener"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "redis_operation_latency",
"help": "The latency of an operation talking to Redis.",
"doc": "{{since('2026.03.04-bb93ecb1')}}\n\nThe `service` key represents the redis server/service. It is not\na direct match to a server name as it is really a hash of the\noverall redis configuration information used in the client.\nIt might look something like:\n`redis://127.0.0.1:24419,redis://127.0.0.1:7779,redis://127.0.0.1:29469-2ce79dd1`\nfor a cluster configuration, or `redis://127.0.0.1:16267-f4da6e64`\nfor a single node cluster configuration.\nYou should anticipate that the `-HEX` suffix can and will change\nin an unspecified way as you vary the redis connection parameters.\n\nThe `operation` key indicates the operation, which can be a `ping`,\na `query` or a `script`.\n\n`status` will be either `ok` or `error` to indicate whether this\nis tracking a successful or failed operation.\n\nSince histograms track a count of operations, you can track the\nrate of `redis_operation_latency_count` where `status=error`\nto have an indication of the failure rate of redis operations.",
"metric_type": "Histogram",
"label_names": [
"service",
"operation",
"status"
],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
},
{
"name": "system_cpu_usage_normalized",
"help": "The sum of the system-wide CPU usage for each CPU in the system, divided by the number of CPUs.",
"doc": "100% in this metric indicates that all CPU cores are 100% busy.\n\nThis metric is scoped to the system, reflecting the total load on the\nsystem, not just from the kumo related process(es).",
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "system_cpu_usage_sum",
"help": "The sum of the system-wide CPU usage for each CPU in the system, can add up to more than 100%.",
"doc": "Each CPU has a value from 0-100% busy; a value of 100% in this metric\nindicates that the load is equivalent to one fully utilized CPU.\n\nA multi-CPU system can report more than 100% in this metric; a dual-CPU\nsystem reporting 200% indicates that both CPUs are fully utilized.\n\nSee system_cpu_usage_normalized for a version of this metric that scales from\n0% (totally idle) to 100% (totally saturated).\n\nThis metric is scoped to the system, reflecting the total load on the\nsystem, not just from the kumo related process(es).",
"metric_type": "Gauge",
"label_names": [],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "thread_pool_parked",
"help": "number of parked (idle) threads in a thread pool",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"pool"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "thread_pool_size",
"help": "number of threads in a thread pool",
"doc": null,
"metric_type": "Gauge",
"label_names": [
"pool"
],
"buckets": [],
"pruning": "NonPruning"
},
{
"name": "user_lua_latency",
"help": "How many seconds something user-defined took to run in your lua policy.",
"doc": "This histogram is updated by policy scripts that employ the\n[kumo.time.start_timer](../../kumo.time/start_timer.md) function\nto record a duration in the policy.\n\nThe `label` is whatever you specified as the label(s) to the various\n`kumo.time.start_timer` calls in the policy.",
"metric_type": "Histogram",
"label_names": [
"label"
],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.5,
5.0,
10.0
],
"pruning": "NonPruning"
}
]