mirror of
https://github.com/quickwit-oss/tantivy.git
synced 2026-01-06 09:12:55 +00:00
56d79cb203
* fix cardinality aggregation performance fix cardinality performance by fetching multiple terms at once. This avoids decompressing the same block and keeps the buffer state between terms. add cardinality aggregation benchmark bump rust version to 1.66 Performance comparison to before (AllQuery) ``` full cardinality_agg Memory: 3.5 MB (-0.00%) Avg: 21.2256ms (-97.78%) Median: 21.0042ms (-97.82%) [20.4717ms .. 23.6206ms] terms_few_with_cardinality_agg Memory: 10.6 MB Avg: 81.9293ms (-97.37%) Median: 81.5526ms (-97.38%) [79.7564ms .. 88.0374ms] dense cardinality_agg Memory: 3.6 MB (-0.00%) Avg: 25.9372ms (-97.24%) Median: 25.7744ms (-97.25%) [24.7241ms .. 27.8793ms] terms_few_with_cardinality_agg Memory: 10.6 MB Avg: 93.9897ms (-96.91%) Median: 92.7821ms (-96.94%) [90.3312ms .. 117.4076ms] sparse cardinality_agg Memory: 895.4 KB (-0.00%) Avg: 22.5113ms (-95.01%) Median: 22.5629ms (-94.99%) [22.1628ms .. 22.9436ms] terms_few_with_cardinality_agg Memory: 680.2 KB Avg: 26.4250ms (-94.85%) Median: 26.4135ms (-94.86%) [26.3210ms .. 26.6774ms] ``` * clippy * assert for sorted ordinals
165 lines
4.5 KiB
TOML
165 lines
4.5 KiB
TOML
[package]
|
|
name = "tantivy"
|
|
version = "0.23.0"
|
|
authors = ["Paul Masurel <paul.masurel@gmail.com>"]
|
|
license = "MIT"
|
|
categories = ["database-implementations", "data-structures"]
|
|
description = """Search engine library"""
|
|
documentation = "https://docs.rs/tantivy/"
|
|
homepage = "https://github.com/quickwit-oss/tantivy"
|
|
repository = "https://github.com/quickwit-oss/tantivy"
|
|
readme = "README.md"
|
|
keywords = ["search", "information", "retrieval"]
|
|
edition = "2021"
|
|
rust-version = "1.66"
|
|
exclude = ["benches/*.json", "benches/*.txt"]
|
|
|
|
[dependencies]
|
|
oneshot = "0.1.7"
|
|
base64 = "0.22.0"
|
|
byteorder = "1.4.3"
|
|
crc32fast = "1.3.2"
|
|
once_cell = "1.10.0"
|
|
regex = { version = "1.5.5", default-features = false, features = [
|
|
"std",
|
|
"unicode",
|
|
] }
|
|
aho-corasick = "1.0"
|
|
tantivy-fst = "0.5"
|
|
memmap2 = { version = "0.9.0", optional = true }
|
|
lz4_flex = { version = "0.11", default-features = false, optional = true }
|
|
zstd = { version = "0.13", optional = true, default-features = false }
|
|
tempfile = { version = "3.3.0", optional = true }
|
|
log = "0.4.16"
|
|
serde = { version = "1.0.136", features = ["derive"] }
|
|
serde_json = "1.0.79"
|
|
fs4 = { version = "0.8.0", optional = true }
|
|
levenshtein_automata = "0.2.1"
|
|
uuid = { version = "1.0.0", features = ["v4", "serde"] }
|
|
crossbeam-channel = "0.5.4"
|
|
rust-stemmers = "1.2.0"
|
|
downcast-rs = "1.2.1"
|
|
bitpacking = { version = "0.9.2", default-features = false, features = [
|
|
"bitpacker4x",
|
|
] }
|
|
census = "0.4.2"
|
|
rustc-hash = "1.1.0"
|
|
thiserror = "1.0.30"
|
|
htmlescape = "0.3.1"
|
|
fail = { version = "0.5.0", optional = true }
|
|
time = { version = "0.3.10", features = ["serde-well-known"] }
|
|
smallvec = "1.8.0"
|
|
rayon = "1.5.2"
|
|
lru = "0.12.0"
|
|
fastdivide = "0.4.0"
|
|
itertools = "0.13.0"
|
|
measure_time = "0.8.2"
|
|
arc-swap = "1.5.0"
|
|
|
|
columnar = { version = "0.3", path = "./columnar", package = "tantivy-columnar" }
|
|
sstable = { version = "0.3", path = "./sstable", package = "tantivy-sstable", optional = true }
|
|
stacker = { version = "0.3", path = "./stacker", package = "tantivy-stacker" }
|
|
query-grammar = { version = "0.22.0", path = "./query-grammar", package = "tantivy-query-grammar" }
|
|
tantivy-bitpacker = { version = "0.6", path = "./bitpacker" }
|
|
common = { version = "0.7", path = "./common/", package = "tantivy-common" }
|
|
tokenizer-api = { version = "0.3", path = "./tokenizer-api", package = "tantivy-tokenizer-api" }
|
|
sketches-ddsketch = { version = "0.3.0", features = ["use_serde"] }
|
|
hyperloglogplus = { version = "0.4.1", features = ["const-loop"] }
|
|
futures-util = { version = "0.3.28", optional = true }
|
|
fnv = "1.0.7"
|
|
|
|
[target.'cfg(windows)'.dependencies]
|
|
winapi = "0.3.9"
|
|
|
|
[dev-dependencies]
|
|
binggan = "0.8.0"
|
|
rand = "0.8.5"
|
|
maplit = "1.0.2"
|
|
matches = "0.1.9"
|
|
pretty_assertions = "1.2.1"
|
|
proptest = "1.0.0"
|
|
test-log = "0.2.10"
|
|
futures = "0.3.21"
|
|
paste = "1.0.11"
|
|
more-asserts = "0.3.1"
|
|
rand_distr = "0.4.3"
|
|
time = { version = "0.3.10", features = ["serde-well-known", "macros"] }
|
|
postcard = { version = "1.0.4", features = [
|
|
"use-std",
|
|
], default-features = false }
|
|
|
|
[target.'cfg(not(windows))'.dev-dependencies]
|
|
criterion = { version = "0.5", default-features = false }
|
|
|
|
[dev-dependencies.fail]
|
|
version = "0.5.0"
|
|
features = ["failpoints"]
|
|
|
|
[profile.release]
|
|
opt-level = 3
|
|
debug = false
|
|
debug-assertions = false
|
|
|
|
[profile.bench]
|
|
opt-level = 3
|
|
debug = true
|
|
debug-assertions = false
|
|
|
|
[profile.test]
|
|
debug-assertions = true
|
|
overflow-checks = true
|
|
|
|
[features]
|
|
default = ["mmap", "stopwords", "lz4-compression"]
|
|
mmap = ["fs4", "tempfile", "memmap2"]
|
|
stopwords = []
|
|
|
|
lz4-compression = ["lz4_flex"]
|
|
zstd-compression = ["zstd"]
|
|
|
|
failpoints = ["fail", "fail/failpoints"]
|
|
unstable = [] # useful for benches.
|
|
|
|
quickwit = ["sstable", "futures-util"]
|
|
|
|
# Compares only the hash of a string when indexing data.
|
|
# Increases indexing speed, but may lead to extremely rare missing terms, when there's a hash collision.
|
|
# Uses 64bit ahash.
|
|
compare_hash_only = ["stacker/compare_hash_only"]
|
|
|
|
[workspace]
|
|
members = [
|
|
"query-grammar",
|
|
"bitpacker",
|
|
"common",
|
|
"ownedbytes",
|
|
"stacker",
|
|
"sstable",
|
|
"tokenizer-api",
|
|
"columnar",
|
|
]
|
|
|
|
# Following the "fail" crate best practises, we isolate
|
|
# tests that define specific behavior in fail check points
|
|
# in a different binary.
|
|
#
|
|
# We do that because, fail rely on a global definition of
|
|
# failpoints behavior and hence, it is incompatible with
|
|
# multithreading.
|
|
[[test]]
|
|
name = "failpoints"
|
|
path = "tests/failpoints/mod.rs"
|
|
required-features = ["failpoints"]
|
|
|
|
[[bench]]
|
|
name = "analyzer"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "index-bench"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "agg_bench"
|
|
harness = false
|