From a2cd6ef9d646d2cbf7151cc86917688a0e20fa74 Mon Sep 17 00:00:00 2001 From: Luca Cominardi Date: Thu, 29 Oct 2026 14:54:52 +0100 Subject: [PATCH] Memoize jitexpr value sources per input term ord A jitexpr value source evaluated its expression once per doc. For an expression of a text input, e.g. `(REGEXP_EXTRACT service "^([^-]+)" 1u64)`, the result only depends on the term of the doc, and text inputs are dictionary encoded with typically few distinct terms per segment (service names, hosts, statuses). Evaluating the expression per doc made a terms aggregation over such a calculated field ~100x slower than over the physical column; on a CloudPrem searcher, the expression accounted for 91% of the CPU of such a query. Memoize the results of an expression of a single text input whose dictionary has at most 65,536 terms, per input term ord. The expression is evaluated on the first doc of each term, and later docs read the encoded result from two dense arrays allocated zeroed when the source is opened (9 bytes per term), plus a slot for docs without a value. Terms are only resolved for the term ords evaluated for the first time. jitexpr functions are deterministic, and `Str` results are interned in the order of the docs as before, so the values and term ords are unchanged. `to_f64` clears the memo, whose results are encoded for the previous result type. Other expressions keep the per-doc path. benches/jitexpr_agg_bench.rs (2M docs, 4 segments, 200 services): terms on (REGEXP_EXTRACT service ..) 140.6 ms -> 7.7 ms (18x) terms on (UPPER service) 305.7 ms -> 8.0 ms (38x) terms on (REGEXP_EXTRACT request_id ..) 1.441 s -> 1.444 s (unique terms, no memo) terms on physical service 1.08 ms -> 1.07 ms --- Cargo.toml | 5 + benches/jitexpr_agg_bench.rs | 93 ++++++ .../jitexpr_value_source/input_column.rs | 43 +++ .../jitexpr_value_source.rs | 268 ++++++++++++++++++ .../jitexpr_value_source/term_resolver.rs | 9 + 5 files changed, 418 insertions(+) create mode 100644 benches/jitexpr_agg_bench.rs diff --git a/Cargo.toml b/Cargo.toml index 67973a152..c50227eab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -202,6 +202,11 @@ harness = false name = "agg_bench" harness = false +[[bench]] +name = "jitexpr_agg_bench" +harness = false +required-features = ["jitexpr"] + [[bench]] name = "exists_json" harness = false diff --git a/benches/jitexpr_agg_bench.rs b/benches/jitexpr_agg_bench.rs new file mode 100644 index 000000000..31566c715 --- /dev/null +++ b/benches/jitexpr_agg_bench.rs @@ -0,0 +1,93 @@ +//! Terms aggregations over jitexpr value sources. +//! +//! The text inputs mimic log attributes: `service` has a few hundred distinct values per segment, +//! like service names, and `request_id` is unique per doc. + +use std::sync::Arc; + +use binggan::plugins::PeakMemAllocPlugin; +use binggan::{black_box, BenchRunner, PeakMemAlloc, INSTRUMENTED_SYSTEM}; +use rand::rngs::StdRng; +use rand::{Rng, SeedableRng}; +use serde_json::json; +use tantivy::aggregation::agg_req::Aggregations; +use tantivy::aggregation::{ + AggContextParams, AggregationCollector, JitExprValueSourceProvider, ValueSourceRegistry, +}; +use tantivy::indexer::NoMergePolicy; +use tantivy::query::AllQuery; +use tantivy::schema::{Schema, FAST, STRING}; +use tantivy::{doc, Index, Searcher}; + +#[global_allocator] +pub static GLOBAL: &PeakMemAlloc = &INSTRUMENTED_SYSTEM; + +const NUM_DOCS: usize = 2_000_000; +const NUM_SEGMENTS: usize = 4; +const NUM_SERVICES: usize = 200; + +fn main() { + let index = create_index(); + let searcher = index.reader().unwrap().searcher(); + let mut runner = BenchRunner::new(); + runner.add_plugin(PeakMemAllocPlugin::new(GLOBAL)); + let mut group = runner.new_group(); + group.set_name("jitexpr_terms"); + for (name, field, expression) in [ + ("terms_physical_service", "service", None), + ( + "terms_regexp_extract_service", + "#first", + Some(r#"(REGEXP_EXTRACT service "^([^-]+)" 1u64)"#), + ), + ("terms_upper_service", "#upper", Some("(UPPER service)")), + ( + "terms_regexp_extract_request_id", + "#first", + Some(r#"(REGEXP_EXTRACT request_id "^([^-]+)" 1u64)"#), + ), + ] { + let mut registry = ValueSourceRegistry::default(); + if let Some(expression) = expression { + let expression = tantivy::jitexpr::ast::deserialize(expression).unwrap(); + let provider = JitExprValueSourceProvider::new(expression).unwrap(); + registry.register(field, Arc::new(provider)); + } + let context = AggContextParams::default().with_value_sources(Arc::new(registry)); + let aggs: Aggregations = + serde_json::from_value(json!({ "t": { "terms": { "field": field, "size": 10 } } })) + .unwrap(); + group.register_with_input(name, &searcher, move |searcher: &Searcher| { + let collector = AggregationCollector::from_aggs(aggs.clone(), context.clone()); + black_box(searcher.search(&AllQuery, &collector).unwrap()); + }); + } + group.run(); +} + +fn create_index() -> Index { + let mut schema_builder = Schema::builder(); + let service = schema_builder.add_text_field("service", STRING | FAST); + let request_id = schema_builder.add_text_field("request_id", STRING | FAST); + let index = Index::create_in_ram(schema_builder.build()); + let mut writer = index.writer_with_num_threads(1, 500_000_000).unwrap(); + writer.set_merge_policy(Box::new(NoMergePolicy)); + let services: Vec = (0..NUM_SERVICES) + .map(|idx| format!("team{}-service{idx}-api", idx % 20)) + .collect(); + let mut rng = StdRng::seed_from_u64(42); + for doc_pos in 0..NUM_DOCS { + let service_value = &services[rng.random_range(0..NUM_SERVICES)]; + writer + .add_document(doc!( + service => service_value.as_str(), + request_id => format!("req{}-{doc_pos}", doc_pos % 7), + )) + .unwrap(); + if (doc_pos + 1) % (NUM_DOCS / NUM_SEGMENTS) == 0 { + writer.commit().unwrap(); + } + } + writer.commit().unwrap(); + index +} diff --git a/src/aggregation/value_source/jitexpr_value_source/input_column.rs b/src/aggregation/value_source/jitexpr_value_source/input_column.rs index dd97dce1f..f27ce9b4e 100644 --- a/src/aggregation/value_source/jitexpr_value_source/input_column.rs +++ b/src/aggregation/value_source/jitexpr_value_source/input_column.rs @@ -61,6 +61,37 @@ impl InputColumn { self.var_type } + /// Returns the number of terms of the dictionary of a `Str` input, or `None` for the other + /// types. + pub(super) fn num_terms(&self) -> Option { + self.term_resolver.num_terms() + } + + /// Loads the first value of each doc of `docs`, without resolving their terms. + /// + /// Afterwards, read the values with `first_value`, and their terms with `value_of`: `value` + /// must not be called before the next `load_block`. + pub(super) fn load_block_values(&mut self, docs: &[DocId]) { + // `first_vals` does not write the entries of the docs without a value: we reset them. + self.first_values.clear(); + self.first_values.resize(docs.len(), None); + self.column.first_vals(docs, &mut self.first_values); + } + + /// Returns the first value of `docs[doc_pos]` in its monotonic `u64` mapping (a term ord for + /// a `Str` input), for the `docs` of the last `load_block_values` call. + #[inline] + pub(super) fn first_value(&self, doc_pos: usize) -> Option { + self.first_values[doc_pos] + } + + /// Returns the first value of `doc` in its monotonic `u64` mapping, without going through a + /// block. + #[inline] + pub(super) fn first_value_for_doc(&self, doc: DocId) -> Option { + self.column.first(doc) + } + /// Loads the first value of each doc of `docs`. /// /// Panics if the dictionary of a text column is corrupted. @@ -99,6 +130,18 @@ impl InputColumn { let Some(value) = self.column.first(doc) else { return VariableValue::none(); }; + self.value_of(value) + } + + /// Returns the variable value of `value`, a value of the column in its monotonic `u64` + /// mapping. Resolves the term of a `Str` input outside of the block path. + /// + /// Invalidates the last `load_block` call: `value` must not be called before the next + /// `load_block`. + /// + /// Panics if the dictionary of a text column is corrupted. + #[inline] + pub(super) fn value_of(&mut self, value: u64) -> VariableValue<'_> { match self.var_type { VarType::Bool => VariableValue::from(bool::from_u64(value)), VarType::I64 => VariableValue::from(i64::from_u64(value)), diff --git a/src/aggregation/value_source/jitexpr_value_source/jitexpr_value_source.rs b/src/aggregation/value_source/jitexpr_value_source/jitexpr_value_source.rs index c15d0f625..a4a0b250b 100644 --- a/src/aggregation/value_source/jitexpr_value_source/jitexpr_value_source.rs +++ b/src/aggregation/value_source/jitexpr_value_source/jitexpr_value_source.rs @@ -11,6 +11,13 @@ use crate::aggregation::f64_from_fastfield_u64; use crate::aggregation::value_source::{coerce_to_f64, ValueSource, ValueSourceDictionary}; use crate::DocId; +/// An expression of a single text input whose dictionary has at most this many terms gets its +/// results memoized per input term ord, see [`TermOrdMemo`]. +#[cfg(not(test))] +const TERM_ORD_MEMO_MAX_NUM_TERMS: usize = 1 << 16; +#[cfg(test)] +const TERM_ORD_MEMO_MAX_NUM_TERMS: usize = 4; + /// The value source of a jitexpr expression on a segment. /// /// Each input takes the first value of the doc. A doc without a value for an input gets null for @@ -23,6 +30,9 @@ use crate::DocId; /// /// Values use the monotonic `u64` mapping of fast fields. `Str` results are interned in a /// [`DynamicTermDictionary`] owned by the source: their values are term ords in first-seen order. +/// +/// An expression of a single text input with a small dictionary is evaluated once per input +/// term instead of once per doc, see [`TermOrdMemo`]. pub(crate) struct JitExprValueSource { compiled_fn: CompiledFnCtx, /// The column type of the result type of `compiled_fn`. @@ -34,6 +44,83 @@ pub(crate) struct JitExprValueSource { /// Reusable allocation for the arguments of `compiled_fn`, lent as a `Vec` for /// each block. args: VersatileBuffer>, + /// The results per input term ord, for an expression of a single text input with a small + /// dictionary. + term_ord_memo: Option, +} + +/// The results of an expression of a single text input, per term ord of the input. +/// +/// jitexpr functions are deterministic, so the result of a doc only depends on the term of its +/// input: the expression is evaluated on the first doc of each term, and later docs reuse the +/// result. Text inputs are dictionary encoded and typically have few distinct terms per segment +/// (service names, hosts, statuses), while evaluating the expression can be expensive (regular +/// expressions). +/// +/// The results are the encoded results: `Str` results are interned when first evaluated, in the +/// order of the docs, so term ords are assigned in the same first-seen order as without the memo. +/// +/// The memo takes 9 bytes per term of the input dictionary, allocated zeroed when the source is +/// opened. It is limited to dictionaries of at most `TERM_ORD_MEMO_MAX_NUM_TERMS` terms: larger +/// dictionaries tend to be identifiers, with few repeated terms. +struct TermOrdMemo { + /// `states[term_ord]` tells whether the result of `term_ord` is unknown, null, or a value. + states: Vec, + /// `values[term_ord]` is the result of `term_ord` if its state is `MEMO_VALUE`. + values: Vec, + /// The result for a doc without an input value: `None` until evaluated. + null_input_result: Option>, +} + +const MEMO_UNKNOWN: u8 = 0; +const MEMO_NULL: u8 = 1; +const MEMO_VALUE: u8 = 2; + +impl TermOrdMemo { + fn new(num_terms: usize) -> TermOrdMemo { + TermOrdMemo { + states: vec![MEMO_UNKNOWN; num_terms], + values: vec![0; num_terms], + null_input_result: None, + } + } + + /// Returns the result for `input_value` if known: `Some(None)` for a null result. + #[inline] + fn get(&self, input_value: Option) -> Option> { + let Some(term_ord) = input_value else { + return self.null_input_result; + }; + match self.states.get(term_ord as usize).copied() { + Some(MEMO_VALUE) => Some(Some(self.values[term_ord as usize])), + Some(MEMO_NULL) => Some(None), + // Unknown, or out of the dictionary (not memoized). + _ => None, + } + } + + fn set(&mut self, input_value: Option, result: Option) { + let Some(term_ord) = input_value else { + self.null_input_result = Some(result); + return; + }; + let Some(state) = self.states.get_mut(term_ord as usize) else { + return; + }; + match result { + Some(value) => { + *state = MEMO_VALUE; + self.values[term_ord as usize] = value; + } + None => *state = MEMO_NULL, + } + } + + /// Forgets all of the results. + fn clear(&mut self) { + self.states.fill(MEMO_UNKNOWN); + self.null_input_result = None; + } } impl JitExprValueSource { @@ -51,15 +138,56 @@ impl JitExprValueSource { assert_eq!(compiled_input.r#type, input.var_type()); } let num_inputs = inputs.len(); + let term_ord_memo = match inputs.as_slice() { + [input] => input + .num_terms() + .filter(|&num_terms| num_terms <= TERM_ORD_MEMO_MAX_NUM_TERMS) + .map(TermOrdMemo::new), + _ => None, + }; JitExprValueSource { compiled_fn, column_type, inputs, term_dictionary: DynamicTermDictionary::default(), args: VersatileBuffer::with_capacity(num_inputs), + term_ord_memo, } } + /// Returns the encoded result for `input_value`, the value of the only input of a memoized + /// expression, evaluating the expression if the result is not memoized yet. + #[inline] + fn memoized_result(&mut self, input_value: Option) -> Option { + let JitExprValueSource { + compiled_fn, + inputs, + term_dictionary, + args, + term_ord_memo, + .. + } = self; + let memo = term_ord_memo + .as_mut() + .expect("only called for a memoized expression"); + if let Some(result) = memo.get(input_value) { + return result; + } + let result_type: VarType = compiled_fn.result_type(); + let mut args = args.borrow::(); + args.push(match input_value { + Some(value) => inputs[0].value_of(value), + None => VariableValue::none(), + }); + // SAFETY: `args` follow the compiled inputs, with their types (see `new`). The string is + // borrowed from `inputs`, which is not modified during the call. + let result: VariableValue = unsafe { compiled_fn.call(&args) }; + // SAFETY: `result` has the result type of the compiled function. + let encoded: Option = unsafe { encode_result(result, result_type, term_dictionary) }; + memo.set(input_value, encoded); + encoded + } + /// Evaluates the expression on `doc`, without going through `load_block`. /// /// Returns the value in the monotonic `u64` mapping (e.g. `true` is 1), or `None` if the @@ -67,6 +195,10 @@ impl JitExprValueSource { /// /// Panics if the dictionary of a text input is corrupted. pub(crate) fn eval_single_doc(&mut self, doc: DocId) -> Option { + if self.term_ord_memo.is_some() { + let input_value = self.inputs[0].first_value_for_doc(doc); + return self.memoized_result(input_value); + } let JitExprValueSource { compiled_fn, inputs, @@ -123,6 +255,22 @@ impl ValueSource for JitExprValueSource { ) -> Cardinality { values.clear(); docids.clear(); + if self.term_ord_memo.is_some() { + // Terms are only resolved for the term ords evaluated for the first time. + self.inputs[0].load_block_values(docs); + for (doc_pos, &doc) in docs.iter().enumerate() { + let input_value = self.inputs[0].first_value(doc_pos); + if let Some(value) = self.memoized_result(input_value) { + values.push(value); + docids.push(doc); + } + } + return if values.len() == docs.len() { + Cardinality::Full + } else { + Cardinality::Optional + }; + } for input in &mut self.inputs { input.load_block(docs); } @@ -174,6 +322,10 @@ impl ValueSource for JitExprValueSource { debug_assert!(coerced_fn.inputs() == self.compiled_fn.inputs()); self.compiled_fn = coerced_fn.context(); self.column_type = ColumnType::F64; + // The memoized results are encoded for the previous result type. + if let Some(memo) = &mut self.term_ord_memo { + memo.clear(); + } self } @@ -454,6 +606,122 @@ mod tests { } } + fn open_variable_source(index: &Index, expression: &str) -> JitExprValueSource { + let provider = + JitExprValueSourceProvider::new(jitexpr::ast::deserialize(expression).unwrap()) + .unwrap(); + let searcher = index.reader().unwrap().searcher(); + match provider + .open_segment(searcher.segment_reader(0), None) + .unwrap() + { + JitExprSegmentSource::Variable(source) => source, + _ => panic!("expected a variable source for `{expression}`"), + } + } + + #[test] + fn test_term_ord_memo_is_limited_to_a_single_small_text_input() { + // At most `TERM_ORD_MEMO_MAX_NUM_TERMS` (4 in tests) terms. + let small = create_index(&["b", "a", "b", "c"], &[]); + assert!(open_variable_source(&small, "(UPPER label)") + .term_ord_memo + .is_some()); + // A numeric input. + assert!(open_variable_source(&small, "(ADD number 1u64)") + .term_ord_memo + .is_none()); + // Several inputs. + assert!( + open_variable_source(&small, r#"(IF (EQ number 1u64) label "x")"#) + .term_ord_memo + .is_none() + ); + let large = create_index(&["a", "b", "c", "d", "e"], &[]); + assert!(open_variable_source(&large, "(UPPER label)") + .term_ord_memo + .is_none()); + } + + #[test] + fn test_term_ord_memo_matches_per_doc_evaluation() { + let index = create_index(&["b_1", "a_2", "b_1", "c", "a_2", "b_1", "c"], &[3, 5]); + let expressions = [ + "(UPPER label)", + r#"(REGEXP_EXTRACT label "^([^_]+)_" 1u64)"#, + r#"(EQ label "b_1")"#, + "(IS_NULL label)", + r#"(IF (EQ label "a_2") 1i64 2i64)"#, + ]; + let blocks: [&[DocId]; 3] = [&[0, 1, 2, 3], &[4, 5, 6], &[6, 0, 3, 1]]; + for expression in expressions { + let mut memoized = open_variable_source(&index, expression); + assert!(memoized.term_ord_memo.is_some(), "`{expression}`"); + let mut per_doc = open_variable_source(&index, expression); + per_doc.term_ord_memo = None; + for docs in blocks { + assert_eq!( + load_block(&mut memoized, docs), + load_block(&mut per_doc, docs), + "`{expression}` on {docs:?}" + ); + } + for doc in 0..7 { + assert_eq!( + memoized.eval_single_doc(doc), + per_doc.eval_single_doc(doc), + "`{expression}` on doc {doc}" + ); + } + if memoized.column_type == ColumnType::Str { + // Same terms, interned in the same first-seen order. + let num_terms = per_doc.term_dictionary.num_terms(); + assert_eq!(memoized.term_dictionary.num_terms(), num_terms); + let all_ords: Vec = (0..num_terms).collect(); + assert_eq!( + terms(&memoized, &all_ords), + terms(&per_doc, &all_ords), + "`{expression}`" + ); + } + } + } + + #[test] + fn test_term_ord_memo_null_input() { + let index = create_index(&["a", "b", "a"], &[1]); + let mut source = open_variable_source(&index, "(IS_NULL label)"); + assert!(source.term_ord_memo.is_some()); + // Doc 1 has no label. The second block only hits the memo. + for _ in 0..2 { + let (cardinality, values, _) = load_block(&mut source, &[0, 1, 2]); + assert_eq!(cardinality, Cardinality::Full); + assert_eq!(values, vec![0, 1, 0]); + } + // A null result for a null input. + let mut source = open_variable_source(&index, "(UPPER label)"); + let (cardinality, values, docids) = load_block(&mut source, &[0, 1, 2]); + assert_eq!(cardinality, Cardinality::Optional); + assert_eq!(values, vec![0, 0]); + assert_eq!(docids, vec![0, 2]); + } + + #[test] + fn test_to_f64_after_memoized_block() { + let index = create_index(&["a", "b", "a"], &[]); + let mut source = open_variable_source(&index, r#"(IF (EQ label "a") 1i64 2i64)"#); + assert!(source.term_ord_memo.is_some()); + // Memoizes the `i64` encodings. + let (_, values, _) = load_block(&mut source, &[0, 1, 2]); + let decoded: Vec = values.iter().map(|value| i64::from_u64(*value)).collect(); + assert_eq!(decoded, vec![1, 2, 1]); + let mut source = Box::new(source).to_f64(); + assert_eq!(source.column_type(), ColumnType::F64); + let (_, values, _) = load_block(&mut *source, &[0, 1, 2]); + let decoded: Vec = values.iter().map(|value| f64::from_u64(*value)).collect(); + assert_eq!(decoded, vec![1.0, 2.0, 1.0]); + } + #[test] fn test_to_f64() { let index = create_index(&["a", "b", "c"], &[1]); diff --git a/src/aggregation/value_source/jitexpr_value_source/term_resolver.rs b/src/aggregation/value_source/jitexpr_value_source/term_resolver.rs index c41b94918..2ccb41099 100644 --- a/src/aggregation/value_source/jitexpr_value_source/term_resolver.rs +++ b/src/aggregation/value_source/jitexpr_value_source/term_resolver.rs @@ -39,6 +39,15 @@ impl TermResolver { ))) } + /// Returns the number of terms of the dictionary, or `None` if this is not a text input. + pub(super) fn num_terms(&self) -> Option { + match self { + TermResolver::None => None, + TermResolver::Full(all_terms) => Some(all_terms.num_terms()), + TermResolver::Cached(cached_terms) => Some(cached_terms.term_dictionary.num_terms()), + } + } + /// Prepares the terms of `term_ords`, the term ords of a block. /// /// Panics if the dictionary is corrupted: `load_block` cannot return an error.