starred/openobserve
1
0
Fork 0
mirror of https://github.com/openobserve/openobserve.git synced 2026-11-04 08:09:27 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
77a4fc5f32e0bc1c8997dc508440bd59cdcf2c4d
openobserve/Cargo.toml
T
Huaijin 9700fd6c43 perf: parallel zstd compress (#12960) 
related to https://github.com/openobserve/o2-enterprise/issues/1860

- [x] limit parallel by the target partitions


## Summary

This PR improves Flight `do_get` response throughput by executing and
encoding output partitions in parallel instead of funneling everything
through a single sequential stream/encoder path.

## Changes

- Switches Flight `do_get` from `execute_stream` to
`execute_stream_partitioned`.
- Adds per-partition encoder tasks that coalesce RecordBatches, encode
Arrow IPC, and apply ZSTD compression independently.
- Splits Flight encoding into:
  - `ChunkCoalescer` for 8192-row batch coalescing.
  - `FlightDataEncoder` for IPC/ZSTD encoding of completed chunks.
- Adds `PartitionEncoderStream` to handle per-partition batch coalescing
and encoding.
- Preserves custom Flight messages behavior:
  - early-emitted messages are sent before the first data message;
  - stats/metrics/peak-memory messages are sent after data completes.
- Uses a fresh dictionary tracker per encoded chunk so
dictionary-encoded columns remain decodable when chunks are encoded
independently.
- Cleans up stream lifecycle handling by aborting partition tasks before
clearing session resources.

## Performance

100M records
single node optimzier disable(to simulate the distribute query)

### Q1
```sql
SELECT
  trace_id,
  count(*) AS cnt,
  max(_timestamp) AS ts
FROM
  "k8s_logs"
GROUP BY
  trace_id ORDER BY cnt DESC LIMIT 100
```
12M trace_id unique value
12M records send from follower to leader
main branch: 6s 
this branch: 3s


### Q2
```sql
SELECT count(distinct request_id) FROM "k8s_logs"
```
100M request_id unique value
100M records send from follower to leader
main branch: 30s
this branch: 11s


### Q3
```sql
SELECT
  kubernetes_namespace_name,
  COUNT(DISTINCT trace_id) AS cnt
FROM
  k8s_logs
GROUP BY
  kubernetes_namespace_name ORDER BY cnt DESC
```
10 kubernetes_namespace_name unique value, 12M trace_id unique value
63M records send from follower to leader
main branch: 21s
this branch: 8.3s

### Q4
```sql
SELECT
  host,
  COUNT(DISTINCT trace_id) AS cnt
FROM
  k8s_logs
GROUP BY
  host ORDER BY cnt DESC
```
2160 host unique value, 12M trace_id unique value
99.57M records send from follower to leader
main branch: 56s
this branch: 20s
2026-07-02 06:07:47 +00:00

518 lines
17 KiB
TOML
Raw Blame History

[package]
description = "OpenObserve is an observability platform that allows you to capture, search, and analyze your logs, metrics, and traces."
edition = "2024"
homepage = "https://openobserve.ai/"
keywords = [
"search",
"log",
"metrics",
"traces",
"analytics",
"openobserve",
"observability",
"open source",
"open-source",
"opensource",
]
license = "AGPL-3.0"
name = "openobserve"
repository = "https://github.com/openobserve/openobserve/"
version = "0.92.0"
publish = false
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[features]
default = []
enterprise = []
vectorscan = []
cloud = []
vortex = ["config/vortex"]
mimalloc = ["dep:mimalloc"]
jemalloc = ["dep:tikv-jemallocator"]
profiling = [
"jemalloc",
"dep:tikv-jemalloc-ctl",
"dep:tikv-jemalloc-sys",
"dep:jemalloc_pprof",
"dep:pprof",
]
tokio-console = ["dep:console-subscriber"]
[profile.release]
debug = false
strip = true
[profile.release-ci]
inherits = "release"
opt-level = 1
codegen-units = 16
debug = false
debug-assertions = true
strip = true
lto = false
incremental = false
[profile.release-prod]
inherits = "release"
codegen-units = 1
lto = "thin"
[profile.release-profiling]
inherits = "release"
debug = true
strip = false
codegen-units = 4
split-debuginfo = "packed"
[dependencies]
axum.workspace = true
axum-extra.workspace = true
axum-server.workspace = true
axum-client-ip.workspace = true
tower.workspace = true
tower-http.workspace = true
http.workspace = true
http-body.workspace = true
http-body-util.workspace = true
hyper.workspace = true
hyper-util.workspace = true
mime_guess = "2.0"
ahash.workspace = true
anyhow.workspace = true
argon2.workspace = true
async-trait.workspace = true
async-recursion.workspace = true
aws-sdk-sns.workspace = true
base64.workspace = true
bitflags = "2.11"
blake3 = { version = "1.8", features = ["rayon"] }
bytes.workspace = true
byteorder.workspace = true
chrono.workspace = true
clap = { version = "4.5", default-features = false, features = [
"std",
"help",
"usage",
"suggestions",
"cargo",
] }
cron.workspace = true
csv = "1.3"
dashmap.workspace = true
datafusion.workspace = true
datafusion-proto.workspace = true
datafusion-functions-aggregate-common.workspace = true
datafusion-functions-json.workspace = true
arrow.workspace = true
arrow-flight.workspace = true
arrow-json.workspace = true
arrow-schema.workspace = true
parquet.workspace = true
object_store.workspace = true
env_logger.workspace = true
faststr.workspace = true
flate2.workspace = true
futures.workspace = true
hashlink.workspace = true
hashbrown.workspace = true
hex.workspace = true
http-auth-basic = "0.3"
itertools.workspace = true
jsonwebtoken = { version = "10.3", features = ["aws_lc_rs"] }
log.workspace = true
maxminddb = "0.27"
md5.workspace = true
memchr.workspace = true
mimalloc = { version = "0.1.48", features = ["v3"], optional = true }
opentelemetry.workspace = true
opentelemetry_sdk.workspace = true
opentelemetry-otlp.workspace = true
opentelemetry-proto.workspace = true
parking_lot.workspace = true
prometheus.workspace = true
promql-parser = { version = "0.6", git = "https://github.com/openobserve/promql-parser", branch = "openobserve" }
prost.workspace = true
prost-wkt-types.workspace = true
proto.workspace = true
prettytable-rs = "0.10.0"
rand.workspace = true
rayon.workspace = true
regex.workspace = true
regex-syntax.workspace = true
reqwest.workspace = true
rquickjs.workspace = true
rust-embed-for-web = "11.3"
rustls.workspace = true
rustls-native-certs.workspace = true
webpki-roots.workspace = true
sea-orm.workspace = true
segment.workspace = true
serde.workspace = true
serde_json.workspace = true
sha256.workspace = true
snafu.workspace = true
snap.workspace = true
sqlparser.workspace = true
sqlx.workspace = true
strum.workspace = true
svix-ksuid.workspace = true
thiserror.workspace = true
time.workspace = true
tikv-jemallocator = { version = "0.6", optional = true, features = [
"profiling",
"stats",
"unprefixed_malloc_on_supported_platforms",
] }
tikv-jemalloc-ctl = { version = "0.6", optional = true, features = [
"use_std",
"stats",
"profiling",
] }
tikv-jemalloc-sys = { version = "0.6", optional = true, features = [
"profiling",
] }
jemalloc_pprof = { version = "0.8", optional = true, features = [
"symbolize",
"flamegraph",
] }
pprof = { version = "0.15", optional = true, features = [
"flamegraph",
"protobuf-codec",
] }
tempfile.workspace = true
tiktoken-rs.workspace = true
tokio.workspace = true
console-subscriber = { version = "0.5", optional = true }
tonic.workspace = true
tonic-prost.workspace = true
tracing.workspace = true
tracing-appender.workspace = true
tracing-opentelemetry.workspace = true
tracing-subscriber.workspace = true
uaparser = "0.6.4"
url.workspace = true
utoipa.workspace = true
utoipa-axum.workspace = true
utoipa-swagger-ui.workspace = true
version-compare = "0.2.0"
vector-enrichment.workspace = true
x509-parser = "0.18.0"
vrl.workspace = true
zstd.workspace = true
config.workspace = true
infra.workspace = true
ingester.workspace = true
wal.workspace = true
report_server.workspace = true
chromiumoxide.workspace = true
lettre.workspace = true
tantivy.workspace = true
tantivy-fst.workspace = true
tantivy_utils.workspace = true
zip.workspace = true
futures-util = "0.3.31"
tokio-util.workspace = true
tokio-stream.workspace = true
mime = "0.3.17"
pin-project-lite.workspace = true
futures-core = "0.3.31"
derive_more = { version = "2.1", features = ["full"] }
brotli = "8.0.2"
tokio-rustls.workspace = true
roaring.workspace = true
flight.workspace = true
async-stream.workspace = true
sourcemap = "9.3.2"
[dev-dependencies]
expect-test.workspace = true
base64 = "0.22"
float-cmp = "0.10"
multer = "3.1"
rcgen = "0.14.4"
[workspace]
members = [
"src/config",
"src/infra",
"src/ingester",
"src/wal",
"src/proto",
"src/report_server",
"src/flight",
"src/tantivy_utils",
]
resolver = "2"
[workspace.package]
version = "0.1.0"
edition = "2024"
license = "AGPL-3.0"
[workspace.dependencies]
config = { path = "src/config" }
infra = { path = "src/infra" }
ingester = { path = "src/ingester" }
wal = { path = "src/wal" }
proto = { path = "src/proto" }
report_server = { path = "src/report_server" }
flight = { path = "src/flight" }
tantivy_utils = { path = "src/tantivy_utils" }
aes-siv = "0.7.0"
ahash = { version = "0.8", features = ["serde"] }
axum = { version = "0.8", features = ["macros", "multipart", "tracing"] }
axum-extra = { version = "0.12", features = [
"typed-header",
"query",
"cookie",
] }
axum-server = { version = "0.8", features = ["tls-rustls"] }
axum-client-ip = "1.3"
async-fs = "2.2"
tower = { version = "0.5", features = ["full"] }
tower-http = { version = "0.6", features = [
"cors",
"compression-gzip",
"compression-br",
"compression-deflate",
"decompression-gzip",
"decompression-br",
"decompression-deflate",
"trace",
"timeout",
"request-id",
"util",
] }
http = "1.4"
http-body = "1.0"
http-body-util = "0.1"
hyper = { version = "1.8", features = ["full"] }
hyper-util = { version = "0.1", features = [
"tokio",
"server",
"http1",
"http2",
] }
anyhow = "1.0"
arc-swap = "1.8"
argon2 = { version = "0.5", features = ["alloc", "password-hash"] }
async-nats = "0.47"
async-stream = "0.3"
async-trait = "0.1"
async-recursion = "1.0"
aws-config = "1.8"
aws-sdk-sns = { version = "1.97", default-features = false, features = [
"behavior-version-latest",
"default-https-client",
"rt-tokio",
] }
base64 = "0.22"
bytes = "1.11"
byteorder = "1.5"
chromiumoxide = { git = "https://github.com/mattsse/chromiumoxide", features = [
"tokio-runtime",
"_fetcher-rusttls-tokio",
], default-features = false, rev = "6f2392f78ae851e2acf33df8e9764cc299d837db" }
chrono = { version = "0.4", default-features = false, features = ["clock"] }
chrono-tz = "0.10"
cityhasher = { version = "0.1", default-features = false }
collapse = "0.1.2"
cron = "0.15"
dashmap = { version = "6.1", features = ["serde"] }
datafusion = "54"
datafusion-proto = "54"
datafusion-functions-aggregate-common = "54"
datafusion-functions-json = { git = "https://github.com/openobserve/datafusion-functions-json", rev = "0df53d710425cc91cf109e77a050bbe1e374e4fe" }
expect-test = "1.4"
arrow = { version = "58", features = ["ipc_compression", "prettyprint"] }
arrow-flight = "58"
arrow-json = "58"
arrow-schema = { version = "58", features = ["serde"] }
parquet = { version = "58", features = ["arrow", "async", "object_store"] }
object_store = { version = "0.13.2", features = ["aws", "azure", "gcp"] }
sqlparser = { version = "0.62", features = ["serde", "visitor"] }
dotenv_config = "0.2"
dotenvy = "0.15"
env_logger = "0.11"
faststr = { version = "0.2", features = ["serde"] }
flate2 = { version = "1.0", features = ["zlib"] }
futures = "0.3"
local-ip-address = "0.6"
hashlink = "0.11"
hashbrown = { version = "0.16", features = ["serde"] }
hex = "0.4"
indexmap = { version = "2.8", features = ["serde"] }
itertools = "0.14"
lettre = { version = "0.11", default-features = false, features = [
"builder",
"hostname",
"smtp-transport",
"pool",
"tokio1",
"tokio1-rustls-tls",
"aws-lc-rs",
] }
log = "0.4"
md5 = "0.8.0"
memchr = "2.7"
murmur3 = "0.5"
ordered-float = { version = "5.1.0", features = ["serde"] }
opentelemetry = "0.32"
opentelemetry_sdk = { version = "0.32", features = [
"experimental_trace_batch_span_processor_with_async_runtime",
"rt-tokio",
"trace",
"metrics",
"spec_unstable_metrics_views",
] }
opentelemetry-otlp = { version = "0.32", features = [
"http-proto",
"serialize",
"serde",
"trace",
"reqwest-client",
"grpc-tonic",
] }
opentelemetry-proto = { version = "0.32", features = [
"gen-tonic",
"with-serde",
"logs",
"metrics",
"trace",
] }
parking_lot = "0.12"
pin-project-lite = "0.2.16"
prometheus = { version = "0.14", features = ["process"] }
prost = "0.14"
prost-wkt-types = "0.7"
rand = "0.10"
rayon = "1.10"
regex = "1.11"
regex-syntax = "0.8"
reqwest = { version = "0.13", default-features = false, features = [
"rustls",
"stream",
"gzip",
"brotli",
"deflate",
"json",
"query",
"form",
] }
roaring = "0.11.3"
rquickjs = { version = "0.11.0", features = ["array-buffer", "classes"] }
rustls = { version = "0.23", default-features = false, features = [
"std",
"tls12",
] }
rustls-native-certs = "0.8"
webpki-roots = "1.0"
sea-orm = { version = "1.1.20", features = [
"sqlx-postgres",
"sqlx-sqlite",
"runtime-tokio-rustls",
"macros",
] }
sea-orm-migration = { version = "1.1.20", features = [
"sqlx-postgres",
"sqlx-sqlite",
"runtime-tokio-rustls",
] }
segment = "~0.2.4"
serde = { version = "1", features = ["derive"] }
serde_json = { version = "1", features = ["arbitrary_precision"] }
sha1 = "0.10.6"
sha256 = "1.6"
snafu = "0.9"
snap = "1"
sqlx = { version = "0.8.6", features = [
"runtime-tokio-rustls",
"postgres",
"sqlite",
"chrono",
] }
strum = { version = "0.28", features = ["derive"] }
svix-ksuid = { version = "0.8", features = ["serde"] }
sysinfo = "0.38"
tantivy = { version = "0.26", features = ["quickwit"] }
tantivy-fst = "0.5"
tempfile = "3"
thiserror = "2.0"
tiktoken-rs = "0.9.1"
time = "0.3"
tokio = { version = "1", features = ["full"] }
tokio-rustls = { version = "0.26", default-features = false, features = [
"logging",
"tls12",
] }
tokio-util = { version = "0.7", features = ["compat"] }
tokio-stream = "0.1"
tonic = { version = "0.14", features = [
"gzip",
"tls-webpki-roots",
"tls-native-roots",
] }
tonic-prost = "0.14"
tracing = "0.1.44"
tracing-appender = "0.2.4"
tracing-log = "0.2"
tracing-opentelemetry = "0.33"
tracing-subscriber = { version = "0.3.22", features = [
"env-filter",
"json",
"registry",
] }
url = "2.5"
urlencoding = "2.1"
utoipa = { version = "5.4", features = ["axum_extras", "openapi_extensions"] }
utoipa-axum = "0.2"
utoipa-swagger-ui = { version = "9", features = ["axum", "vendored"] }
uuid = { version = "1.20", features = ["v7"] }
vector-enrichment = { version = "0.1.0", package = "enrichment", git = "https://github.com/openobserve/vector", branch = "branch-v0.54" }
vrl = { version = "0.31", features = [
"arbitrary",
"value",
"compiler",
"test",
] }
zip = "8.0"
zstd = "0.13"
[patch.crates-io]
datafusion = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-catalog = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-catalog-listing = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-common = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-common-runtime = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-datasource = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-datasource-arrow = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-datasource-csv = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-datasource-json = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-datasource-parquet = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-doc = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-execution = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-expr = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-expr-common = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-functions = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-functions-aggregate = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-functions-aggregate-common = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-functions-nested = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-functions-table = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-functions-window = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-functions-window-common = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-macros = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-optimizer = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-physical-expr = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-physical-expr-adapter = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-physical-expr-common = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-physical-optimizer = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-physical-plan = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-proto = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-proto-common = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-pruning = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-session = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
datafusion-sql = { git = "https://github.com/openobserve/datafusion", rev = "869ec1a70482e1a26c4ac2fde3e9fd5545c33d7b" }
object_store = { git = "https://github.com/openobserve/arrow-rs-object-store", branch = "openobserve-v0.13.2" }
proc-macro-error2 = { git = "https://github.com/hengfeiyang/proc-macro-error-2", branch = "main" }
tantivy = { git = "https://github.com/openobserve/tantivy", branch = "perf/openobserve" }
Reference in New Issue View Git Blame Copy Permalink