mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-10-23 12:22:35 +00:00
Lei, HUANG
52554db52f
* perf(mito2): keep short series unsplit in compaction merges Compaction writes every merged batch to the output SST. Splitting short series before the merge made it emit one small batch per series and input file, which the SST writer and indexer then handled one by one. Compaction now splits only when series are long enough to fill most of a batch. With 16 L0 files of 10k series x 30 rows, one output went from 160,560 merged batches to 592, and merging plus writing took 1.66s instead of 2.67s. Signed-off-by: Lei, HUANG <ratuthomm@gmail.com> * test(mito2): benchmark compacting L0 files into one output The benchmark writes 16 L0 files through the engine, each holding every series over its own time span as consecutive flushes do, and times a strict window compaction that merges them into one output. Every iteration compacts a fresh copy of the same files. With 10k series of 30 rows per file, keeping short series unsplit in compaction merges cut the time from 830ms to 520ms. With 60 series of 5,000 rows per file, which compaction still splits, both took 266ms. Signed-off-by: Lei, HUANG <ratuthomm@gmail.com> --------- Signed-off-by: Lei, HUANG <ratuthomm@gmail.com>
162 lines
4.0 KiB
TOML
162 lines
4.0 KiB
TOML
[package]
|
|
name = "mito2"
|
|
version.workspace = true
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
|
|
[features]
|
|
default = []
|
|
test = ["common-test-util", "rstest", "rstest_reuse", "rskafka"]
|
|
testing = ["test"]
|
|
test-shared-fs-region-migration = []
|
|
enterprise = []
|
|
hdfs-object-store = ["object-store/hdfs-object-store"]
|
|
|
|
[lints]
|
|
workspace = true
|
|
|
|
[dependencies]
|
|
api.workspace = true
|
|
aquamarine.workspace = true
|
|
arrow-schema.workspace = true
|
|
arrow-select.workspace = true
|
|
async-channel = "1.9"
|
|
async-stream.workspace = true
|
|
async-trait.workspace = true
|
|
base64.workspace = true
|
|
bytemuck.workspace = true
|
|
bytes.workspace = true
|
|
chrono.workspace = true
|
|
common-base.workspace = true
|
|
common-datasource.workspace = true
|
|
common-error.workspace = true
|
|
common-function.workspace = true
|
|
common-grpc.workspace = true
|
|
common-macro.workspace = true
|
|
common-memory-manager.workspace = true
|
|
common-meta.workspace = true
|
|
common-query.workspace = true
|
|
common-recordbatch.workspace = true
|
|
common-runtime.workspace = true
|
|
common-stat.workspace = true
|
|
common-telemetry.workspace = true
|
|
common-test-util = { workspace = true, optional = true }
|
|
common-time.workspace = true
|
|
common-wal.workspace = true
|
|
crc32fast = "1"
|
|
crossbeam-utils.workspace = true
|
|
dashmap.workspace = true
|
|
datafusion.workspace = true
|
|
datafusion-common.workspace = true
|
|
datafusion-expr.workspace = true
|
|
datatypes.workspace = true
|
|
derive_more.workspace = true
|
|
dotenv.workspace = true
|
|
either.workspace = true
|
|
futures.workspace = true
|
|
greptime-proto.workspace = true
|
|
humantime.workspace = true
|
|
humantime-serde.workspace = true
|
|
index.workspace = true
|
|
itertools.workspace = true
|
|
lazy_static = "1.4"
|
|
log-store = { workspace = true }
|
|
mito-codec.workspace = true
|
|
moka = { workspace = true, features = ["sync", "future"] }
|
|
object-store = { workspace = true, features = ["testing"] }
|
|
parquet = { workspace = true, features = ["async", "variant_experimental"] }
|
|
partition.workspace = true
|
|
pin-project.workspace = true
|
|
prometheus.workspace = true
|
|
prost.workspace = true
|
|
puffin.workspace = true
|
|
rand.workspace = true
|
|
rayon = "1.10"
|
|
regex.workspace = true
|
|
rskafka = { workspace = true, optional = true }
|
|
rstest = { workspace = true, optional = true }
|
|
rstest_reuse = { workspace = true, optional = true }
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
serde_with.workspace = true
|
|
smallvec.workspace = true
|
|
snafu.workspace = true
|
|
store-api.workspace = true
|
|
strum.workspace = true
|
|
table.workspace = true
|
|
tokio.workspace = true
|
|
tokio-stream.workspace = true
|
|
tokio-util.workspace = true
|
|
tracing.workspace = true
|
|
uuid.workspace = true
|
|
winner_tree.workspace = true
|
|
zstd.workspace = true
|
|
|
|
[dev-dependencies]
|
|
common-function.workspace = true
|
|
common-meta = { workspace = true, features = ["testing"] }
|
|
common-test-util.workspace = true
|
|
criterion = { workspace = true, features = ["async", "async_tokio"] }
|
|
dotenv.workspace = true
|
|
log-store = { workspace = true, features = ["testing"] }
|
|
mito-codec = { workspace = true, features = ["testing"] }
|
|
object-store = { workspace = true, features = ["services-memory"] }
|
|
rskafka.workspace = true
|
|
rstest.workspace = true
|
|
rstest_reuse.workspace = true
|
|
toml.workspace = true
|
|
|
|
[[bench]]
|
|
name = "memtable_bench"
|
|
harness = false
|
|
required-features = ["test"]
|
|
|
|
[[bench]]
|
|
name = "bench_cache_stream"
|
|
harness = false
|
|
required-features = ["test"]
|
|
|
|
[[bench]]
|
|
name = "bench_filter_time_partition"
|
|
harness = false
|
|
required-features = ["test"]
|
|
|
|
[[bench]]
|
|
name = "bench_compaction_picker"
|
|
harness = false
|
|
required-features = ["testing"]
|
|
|
|
[[bench]]
|
|
name = "bench_flat_merge"
|
|
harness = false
|
|
required-features = ["testing"]
|
|
|
|
[[bench]]
|
|
name = "simple_bulk_memtable"
|
|
harness = false
|
|
required-features = ["test"]
|
|
|
|
[[bench]]
|
|
name = "bench_wal_encode"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "bench_index_update"
|
|
harness = false
|
|
required-features = ["testing"]
|
|
|
|
[[bench]]
|
|
name = "bench_pk_tag_column"
|
|
harness = false
|
|
required-features = ["testing"]
|
|
|
|
[[bench]]
|
|
name = "bench_dense_index_update"
|
|
harness = false
|
|
required-features = ["testing"]
|
|
|
|
[[bench]]
|
|
name = "bench_compaction_merge"
|
|
harness = false
|
|
required-features = ["test"]
|