mirror of
https://github.com/mailscope/kumomta.git
synced 2026-10-29 01:36:14 +00:00
Wez Furlong
6d6e6038db
This commit introduces a processor pool with n-cpus as the number of threads as the way that we ingest log records. The ingestion is done via a batch-channel which will allow the potential for automatic batching into larger transactions in a later commit. This commit causes publish requests to return as soon as the record is queued into the channel, rather than once the record has been processed. This means that there is now a window of opportunity where, if tsa-daemon were to panic or be killed, the associated records will be forgotten. There is a maximum of 128k records that can be in the queue at one time, which puts an upper bound on this. In practice, it is not critical if records are lost in this way: it is not significantly different from the time window where a node might continue to send mail before the effect of an automation kicks in. The net effect of this is that the submission latency is now much lower while the system remains un-saturated, but will then block additional submissions until the backlog is cleared.
199 lines
5.6 KiB
TOML
199 lines
5.6 KiB
TOML
[workspace]
|
|
members = [
|
|
"crates/bounce-classify",
|
|
"crates/cidr-map",
|
|
"crates/domain-map",
|
|
"crates/integration-tests",
|
|
"crates/kcli",
|
|
"crates/kumo-chrono-helper",
|
|
"crates/kumo-prometheus",
|
|
"crates/kumod",
|
|
"crates/mailparsing",
|
|
"crates/mod-uuid",
|
|
"crates/mta-sts",
|
|
"crates/proxy-server",
|
|
"crates/regex-set-map",
|
|
"crates/rfc5321",
|
|
"crates/kumo-spf",
|
|
"crates/spool",
|
|
"crates/spool-util",
|
|
"crates/tailer",
|
|
"crates/throttle",
|
|
"crates/timeq",
|
|
"crates/tls-probe",
|
|
"crates/toml2jsonc",
|
|
"crates/traffic-gen",
|
|
"crates/tsa-daemon",
|
|
"crates/uuid-helper",
|
|
"crates/validate-bounces",
|
|
"crates/validate-shaping",
|
|
"crates/version-info",
|
|
"crates/kumo-dmarc",
|
|
]
|
|
resolver = "2"
|
|
|
|
[profile.release]
|
|
opt-level = 3
|
|
debug = 1 # include line tables for more meaningful backtraces
|
|
|
|
[workspace.dependencies]
|
|
amqprs = {version="2.0", features=["tls", "traces"]}
|
|
anyhow = "1.0"
|
|
arc-swap = "1.6"
|
|
async-channel = "2"
|
|
async-stream = "0.3"
|
|
async-trait = "0.1"
|
|
axum = "0.7"
|
|
axum-client-ip = "0.6"
|
|
axum-server = "0.7"
|
|
axum-streams = {version="0.19", features=["text"]}
|
|
backtrace = "0.3"
|
|
batch-channel = "0.4.2"
|
|
bitflags = "2.3"
|
|
bitstring = "0.2"
|
|
bytes = "1.0"
|
|
camino = "1.1"
|
|
caps = "0.5"
|
|
cgroups-rs = "0.3"
|
|
charset = "0.1"
|
|
chrono = { version = "0.4.26", default-features = false }
|
|
chrono-tz = {version="0.10", features=["serde"]}
|
|
cidr = {version="0.3", features=["serde", "bitstring"]}
|
|
clap = {version="4.5", features=["derive", "wrap_help"]}
|
|
clap-markdown = "0.1"
|
|
criterion = "0.5"
|
|
crossbeam-queue= "0.3"
|
|
crossbeam-skiplist= "0.1"
|
|
crossterm = { version = "0.27.0", features = ["event-stream"] }
|
|
data-encoding = "2.6"
|
|
data-encoding-macro = "0.1"
|
|
deadpool = {version="0.12", features=["rt_tokio_1"]}
|
|
ed25519-dalek = "2.0"
|
|
env_logger = "0.11"
|
|
fancy-regex = "0.14"
|
|
filenamegen = "0.2.7"
|
|
flume = "0.11"
|
|
foreign-types = "0.3"
|
|
futures = "0.3.28"
|
|
futures-util = "0.3"
|
|
futures-lite = "2.3"
|
|
gcd = "2.3"
|
|
gethostname = "0.5"
|
|
getrandom = "0.2"
|
|
git2 = { version = "0.19", default-features = false }
|
|
hdrhistogram = "7.5"
|
|
hex = "0.4"
|
|
# Be sure to update the link to the docs in docs/reference/kumo.dns/configure_resolver.md
|
|
# to match the version that we are using when you update this dep
|
|
hickory-resolver = "0.24"
|
|
hickory-proto = "0.24"
|
|
hierarchical_hash_wheel_timer = {version="1.2", default-features=false, features=["fx-hash"]}
|
|
human_bytes = "0.4.3"
|
|
humanize-rs = "0.1"
|
|
humantime = "2.1"
|
|
humansize = "2.1"
|
|
incr_stats = "1.0"
|
|
indexmap = "2.2"
|
|
indoc = "2.0.5"
|
|
instant-xml = "0.5"
|
|
json_comments = "0.2"
|
|
jwalk = "0.8"
|
|
k9 = "0.12"
|
|
lapin = {version="2.5", default-features=false, features=["native-tls"]}
|
|
lexicmp = "0.1"
|
|
libc = "0.2.139"
|
|
libunbound = {git="https://github.com/KumoCorp/libunbound-rs.git", rev="514258fa097d04236761fff2f97b4221ff4965e0"}
|
|
lipsum = "0.9"
|
|
log = "0.4"
|
|
lru-cache = "0.1"
|
|
mac_address = "1.1"
|
|
mail-builder = "0.2"
|
|
mailparse = "0.14"
|
|
maplit = "1.0"
|
|
map_vec = "0.6"
|
|
memchr = "2.5"
|
|
memmap2 = "0.9"
|
|
metrics = "0.24"
|
|
metrics-prometheus = "0.8"
|
|
metrics-tracing-context = "0.17"
|
|
minijinja = {version="2.0.1",features=["loader", "builtins", "json"]}
|
|
minijinja-contrib = {version="2.0.1",features=["datetime", "timezone"]}
|
|
mlua = "0.9"
|
|
nix = "0.28"
|
|
nom = "7.1.0"
|
|
nom_locate = "4.1"
|
|
notify = "7"
|
|
num-format = "0.4.4"
|
|
openssl = { version="=0.10.65" } # pinned; see patch below
|
|
openssl-sys = { version="0.9" }
|
|
ordermap = {version="0.5", features=["serde"]}
|
|
parking_lot = "0.12"
|
|
paste = "1"
|
|
percent-encoding = "2.3"
|
|
pest = "2.7"
|
|
pest_derive = "2.7"
|
|
ppp = "2.2"
|
|
prometheus = "0.13"
|
|
psl = "2.1.46"
|
|
quoted_printable = "0.5"
|
|
rand = "0.8"
|
|
ratatui = { version = "0.26.2", features = ["crossterm", "unstable-widget-ref"] }
|
|
rcgen = "0.13"
|
|
rdkafka = "0.36"
|
|
redis = {version="0.27", features=["cluster", "cluster-async", "connection-manager", "tokio-comp", "tls-native-tls", "tokio-native-tls-comp"]}
|
|
regex = "1.7"
|
|
reqwest = {version="0.12", default-features=false, features=["rustls-tls"]}
|
|
ring = "0.17"
|
|
rocksdb = {version="0.22", features=["jemalloc"]}
|
|
rustls = "0.23"
|
|
rustls-platform-verifier = "0.4"
|
|
self_cell = "1.0"
|
|
serde = {version="1.0", features=["derive"]}
|
|
serde_json = "1.0"
|
|
serde_path_to_error = "0.1"
|
|
serde_with = "3.4"
|
|
serde_yaml = "0.9"
|
|
sha-1 = { version = "0.10", features = ["oid"] }
|
|
sha2 = "0.10"
|
|
slog = "2.7"
|
|
socksv5 = {version="0.3", default-features=false, features=["tokio"]}
|
|
sqlite = "0.36"
|
|
strum = { version = "0.26", features = ["derive"] }
|
|
tabout = "0.3"
|
|
tempfile = "3.10"
|
|
testcontainers-modules = { version = "0.11", default-features = false }
|
|
textwrap = "0.16"
|
|
thiserror = "1"
|
|
tikv-jemallocator = "0.5"
|
|
tikv-jemalloc-sys = {version="0.5", features=["profiling", "unprefixed_malloc_on_supported_platforms"]}
|
|
tokio = "1.32"
|
|
tokio-executor-trait = "2.1.0"
|
|
tokio-metrics = "0.3.1"
|
|
tokio-metrics-collector = "0.2.1"
|
|
tokio-openssl = "0.6.4"
|
|
tokio-reactor-trait = "1.1.0"
|
|
tokio-rustls = "0.26"
|
|
tokio-splice = "0.2"
|
|
tokio-tungstenite = "0.24"
|
|
toml = "0.8"
|
|
toml_edit = "0.22"
|
|
tower-http = {version="0.6", features=["trace", "compression-deflate", "compression-gzip"]}
|
|
tracing = "0.1"
|
|
tracing-appender = "0.2"
|
|
tracing-subscriber = {version="0.3", features=["env-filter", "std", "fmt", "json"]}
|
|
utoipa = {version="4", features=["axum_extras", "time", "uuid"]}
|
|
utoipa-rapidoc = { version="4.0", features = ["axum"] }
|
|
url = "2.4"
|
|
uuid = "1.4"
|
|
vaultrs = "0.7"
|
|
walkdir = "2.3"
|
|
which = "7"
|
|
zstd = "0.13"
|
|
zstd-safe = {version="7.0", features=["std"]}
|
|
|
|
[patch.crates-io]
|
|
# Can be removed once dane support is upstreamed
|
|
openssl = { version="0.10.65", git="https://github.com/KumoCorp/rust-openssl.git", branch="dane" }
|
|
# Can be removed once dane support is upstreamed
|
|
openssl-sys = { version="0.9", git="https://github.com/KumoCorp/rust-openssl.git", branch="dane" }
|