From 6e79b1ace0bc6a22a62f00a58acb4515634a33bc Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Tue, 13 Oct 2026 08:33:55 +0200 Subject: [PATCH] Support plain-encoded text fields in JitExpr and aggregations JitExpr inputs can be plain-encoded text columns, for predicates and calculated fields. Terms and cardinality aggregations read plain text columns through the identity jitexpr expression, which interns the values. Options relying on sorted term ordinals, missing values or multivalued plain columns return an error. --- src/aggregation/accessor_helpers.rs | 62 ++++- src/aggregation/agg_data.rs | 13 +- .../jitexpr_value_source/input_column.rs | 207 +++++++++++++---- .../value_source/jitexpr_value_source/mod.rs | 213 ++++++++++++++++++ .../doc_predicate_query/jitexpr_predicate.rs | 46 ++++ 5 files changed, 494 insertions(+), 47 deletions(-) diff --git a/src/aggregation/accessor_helpers.rs b/src/aggregation/accessor_helpers.rs index 9c6d54ebd..dedf620f8 100644 --- a/src/aggregation/accessor_helpers.rs +++ b/src/aggregation/accessor_helpers.rs @@ -2,7 +2,9 @@ use std::io; -use columnar::{BytesColumn, Column, ColumnType, DynamicColumn, DynamicColumnHandle, StrColumn}; +use columnar::{ + BytesColumn, Column, ColumnType, DynamicColumn, DynamicColumnHandle, StrColumn, StrColumnCodec, +}; use crate::aggregation::value_source::ValueSource; use crate::aggregation::{f64_to_fastfield_u64, Key, ValueSourceRegistry}; @@ -144,6 +146,20 @@ pub(crate) fn get_all_value_sources( Ok(sources) } +#[cfg(feature = "jitexpr")] +use crate::aggregation::jitexpr_value_source::plain_str_value_source; + +/// Aggregations on plain-encoded text columns require the `jitexpr` feature. +#[cfg(not(feature = "jitexpr"))] +fn plain_str_value_source( + _reader: &SegmentReader, + _handle: &DynamicColumnHandle, +) -> crate::Result> { + Err(crate::TantivyError::InvalidArgument( + "aggregations on plain-encoded text fast fields require the `jitexpr` feature".to_string(), + )) +} + /// Builds a value source without any value, with the given type. /// /// `num_rows` is the number of rows of the column (the `max_doc` of the segment). @@ -183,7 +199,7 @@ fn open_physical_column_value_sources( continue; } } - if let Some(column) = open_physical_column_value_source(handle)? { + if let Some(column) = open_physical_column_value_source(reader, handle)? { sources.push(column); if first_only { break; @@ -196,10 +212,22 @@ fn open_physical_column_value_sources( /// Opens a column as a value source. /// /// Returns `None` if the column cannot be read as `u64`. +/// +/// A plain-encoded text column has no term ordinals: it is exposed through a jitexpr source +/// interning its values, see `plain_str_value_source`. fn open_physical_column_value_source( + reader: &SegmentReader, column_handle: DynamicColumnHandle, ) -> crate::Result>> { let column_type = column_handle.column_type(); + if column_handle.str_codec() == Some(StrColumnCodec::Plain) { + if column_type != ColumnType::Str { + return Err(crate::TantivyError::InvalidArgument(format!( + "aggregations are not supported on plain-encoded {column_type} fast fields" + ))); + } + return plain_str_value_source(reader, &column_handle).map(Some); + } if column_type == ColumnType::Str { let DynamicColumn::Str(str_column) = column_handle.open()? else { return Err(crate::TantivyError::InternalError( @@ -213,3 +241,33 @@ fn open_physical_column_value_source( }; Ok(Some(Box::new((column, column_type)))) } + +#[cfg(all(test, not(feature = "jitexpr")))] +mod tests { + use serde_json::json; + + use crate::aggregation::agg_req::Aggregations; + use crate::aggregation::AggregationCollector; + use crate::query::AllQuery; + use crate::schema::{Schema, StrColumnCodec, TextOptions}; + use crate::Index; + + #[test] + fn test_aggregation_on_plain_text_field_requires_jitexpr() { + let mut schema_builder = Schema::builder(); + let label = schema_builder.add_text_field( + "label", + TextOptions::default().set_fast_codec(StrColumnCodec::Plain), + ); + let index = Index::create_in_ram(schema_builder.build()); + let mut writer = index.writer_for_tests().unwrap(); + writer.add_document(doc!(label => "a")).unwrap(); + writer.commit().unwrap(); + let aggs: Aggregations = + serde_json::from_value(json!({ "t": { "terms": { "field": "label" } } })).unwrap(); + let collector = AggregationCollector::from_aggs(aggs, Default::default()); + let searcher = index.reader().unwrap().searcher(); + let err = searcher.search(&AllQuery, &collector).unwrap_err(); + assert!(err.to_string().contains("`jitexpr` feature"), "{err}"); + } +} diff --git a/src/aggregation/agg_data.rs b/src/aggregation/agg_data.rs index 8637380ca..9d87968de 100644 --- a/src/aggregation/agg_data.rs +++ b/src/aggregation/agg_data.rs @@ -292,7 +292,8 @@ fn missing_value_for_source( Some(column) => column.max_value(), None if column_type == ColumnType::Str => { return Err(crate::TantivyError::InvalidArgument(format!( - "`missing` is not supported for the computed text value source `{field_name}`" + "`missing` is not supported for the computed or plain-encoded text field \ + `{field_name}`" ))); } None => 0, @@ -311,7 +312,7 @@ fn require_physical_column( ) -> crate::Result> { source.as_physical_column().cloned().ok_or_else(|| { crate::TantivyError::InvalidArgument(format!( - "{agg_kind} does not support the computed value source `{field_name}`" + "{agg_kind} does not support the computed or plain-encoded field `{field_name}`" )) }) } @@ -326,7 +327,7 @@ fn reject_registered_source( ) -> crate::Result<()> { if value_sources.get(field_name).is_some() { return Err(crate::TantivyError::InvalidArgument(format!( - "{agg_kind} does not support the computed value source `{field_name}`" + "{agg_kind} does not support the computed or plain-encoded field `{field_name}`" ))); } Ok(()) @@ -1003,7 +1004,7 @@ fn build_terms_or_cardinality_nodes( let Some(str_col) = accessor.as_physical_str_column() else { return Err(crate::TantivyError::InvalidArgument(format!( "terms aggregation with `include` / `exclude` requires a physical \ - text field, but `{field_name}` is a computed value source" + text field, but `{field_name}` is a computed or plain-encoded field" ))); }; allowed_term_ids = build_allowed_term_ids_for_str( @@ -1026,14 +1027,14 @@ fn build_terms_or_cardinality_nodes( { return Err(crate::TantivyError::InvalidArgument(format!( "terms aggregation ordered by `_key` is not supported on the computed \ - value source `{field_name}`" + or plain-encoded text field `{field_name}`" ))); } // `min_doc_count: 0` requires listing all of the possible values. if req_internal.min_doc_count == 0 { return Err(crate::TantivyError::InvalidArgument(format!( "terms aggregation with `min_doc_count: 0` is not supported on the \ - computed value source `{field_name}`" + computed or plain-encoded field `{field_name}`" ))); } } diff --git a/src/aggregation/value_source/jitexpr_value_source/input_column.rs b/src/aggregation/value_source/jitexpr_value_source/input_column.rs index dd97dce1f..401873df3 100644 --- a/src/aggregation/value_source/jitexpr_value_source/input_column.rs +++ b/src/aggregation/value_source/jitexpr_value_source/input_column.rs @@ -1,7 +1,10 @@ //! The fast-field columns bound to the inputs of a compiled jitexpr expression. +use std::ops::Range; + use columnar::{ Column, ColumnIndex, DynamicColumn, DynamicColumnHandle, MonotonicallyMappableToU64, + PlainStrColumn, PlainValueReader, }; use jitexpr::types::{VarType, VariableValue}; @@ -12,12 +15,21 @@ use crate::{DocId, TantivyError, COLLECT_BLOCK_BUFFER_LEN}; /// A fast-field column bound to an input of the compiled expression. pub(super) struct InputColumn { var_type: VarType, - /// Values in their monotonic `u64` mapping. Term ords for a `Str` input. - column: Column, - /// `first_values[i]` is the first value of `docs[i]`, for the docs of the last block. - first_values: Vec>, - /// The terms of `first_values`. Not `TermResolver::None` if and only if `var_type` is `Str`. - term_resolver: TermResolver, + values: InputValues, +} + +enum InputValues { + /// Values in their monotonic `u64` mapping. Term ords for a dictionary encoded `Str` input. + Ordinals { + column: Column, + /// `first_values[i]` is the first value of `docs[i]`, for the docs of the last block. + first_values: Vec>, + /// The terms of `first_values`. Not `TermResolver::None` if and only if `var_type` is + /// `Str`. + term_resolver: TermResolver, + }, + /// A plain-encoded `Str` input: the values are decoded from the column. + PlainStr(PlainStrInput), } impl InputColumn { @@ -30,31 +42,39 @@ impl InputColumn { ))); }; if var_type == VarType::Str { - let DynamicColumn::Str(str_column) = handle.open()? else { - return Err(TantivyError::InternalError( - "a text column could not be opened as a text column".to_string(), - )); + let values = match handle.open()? { + DynamicColumn::Str(str_column) => { + let (term_dictionary, term_ords) = str_column.into_parts(); + InputValues::Ordinals { + column: term_ords, + first_values: Vec::with_capacity(COLLECT_BLOCK_BUFFER_LEN), + term_resolver: TermResolver::for_dictionary(term_dictionary)?, + } + } + DynamicColumn::PlainStr(plain_str_column) => { + InputValues::PlainStr(PlainStrInput::new(&plain_str_column)) + } + _ => { + return Err(TantivyError::InternalError( + "a text column could not be opened as a text column".to_string(), + )); + } }; - let (term_dictionary, term_ords) = str_column.into_parts(); - Ok(InputColumn { - var_type, - column: term_ords, - first_values: Vec::with_capacity(COLLECT_BLOCK_BUFFER_LEN), - term_resolver: TermResolver::for_dictionary(term_dictionary)?, - }) - } else { - let column = handle.open_u64_lenient()?.ok_or_else(|| { - TantivyError::InternalError(format!( - "a column of type {column_type} could not be opened as u64" - )) - })?; - Ok(InputColumn { - var_type, + return Ok(InputColumn { var_type, values }); + } + let column = handle.open_u64_lenient()?.ok_or_else(|| { + TantivyError::InternalError(format!( + "a column of type {column_type} could not be opened as u64" + )) + })?; + Ok(InputColumn { + var_type, + values: InputValues::Ordinals { column, first_values: Vec::with_capacity(COLLECT_BLOCK_BUFFER_LEN), term_resolver: TermResolver::None, - }) - } + }, + }) } pub(super) fn var_type(&self) -> VarType { @@ -63,19 +83,43 @@ impl InputColumn { /// Loads the first value of each doc of `docs`. /// - /// Panics if the dictionary of a text column is corrupted. + /// Panics if the dictionary of a text column is corrupted, or if a plain column cannot be + /// read. pub(super) fn load_block(&mut self, docs: &[DocId]) { - // `first_vals` does not write the entries of the docs without a value: we reset them. - self.first_values.clear(); - self.first_values.resize(docs.len(), None); - self.column.first_vals(docs, &mut self.first_values); - self.term_resolver.load(&self.first_values); + match &mut self.values { + InputValues::Ordinals { + column, + first_values, + term_resolver, + } => { + // `first_vals` does not write the entries of the docs without a value: we reset + // them. + first_values.clear(); + first_values.resize(docs.len(), None); + column.first_vals(docs, first_values); + term_resolver.load(first_values); + } + InputValues::PlainStr(plain_str_input) => plain_str_input.load_block(docs), + } } /// Returns the value of `docs[doc_pos]`, for the `docs` of the last `load_block` call. #[inline] pub(super) fn value(&self, doc_pos: usize) -> VariableValue<'_> { - let Some(value) = self.first_values[doc_pos] else { + let (first_values, term_resolver) = match &self.values { + InputValues::Ordinals { + first_values, + term_resolver, + .. + } => (first_values, term_resolver), + InputValues::PlainStr(plain_str_input) => { + return plain_str_input + .value(doc_pos) + .map(VariableValue::from) + .unwrap_or(VariableValue::none()); + } + }; + let Some(value) = first_values[doc_pos] else { return VariableValue::none(); }; match self.var_type { @@ -83,7 +127,7 @@ impl InputColumn { VarType::I64 => VariableValue::from(i64::from_u64(value)), VarType::U64 => VariableValue::from(value), VarType::F64 => VariableValue::from(f64::from_u64(value)), - VarType::Str => VariableValue::from(self.term_resolver.term(doc_pos, value)), + VarType::Str => VariableValue::from(term_resolver.term(doc_pos, value)), VarType::None => VariableValue::none(), } } @@ -93,10 +137,24 @@ impl InputColumn { /// Invalidates the last `load_block` call: `value` must not be called before the next /// `load_block`. /// - /// Panics if the dictionary of a text column is corrupted. + /// Panics if the dictionary of a text column is corrupted, or if a plain column cannot be + /// read. #[inline] pub(super) fn value_for_doc(&mut self, doc: DocId) -> VariableValue<'_> { - let Some(value) = self.column.first(doc) else { + let (column, term_resolver) = match &mut self.values { + InputValues::Ordinals { + column, + term_resolver, + .. + } => (column, term_resolver), + InputValues::PlainStr(plain_str_input) => { + return plain_str_input + .value_for_doc(doc) + .map(VariableValue::from) + .unwrap_or(VariableValue::none()); + } + }; + let Some(value) = column.first(doc) else { return VariableValue::none(); }; match self.var_type { @@ -104,13 +162,84 @@ impl InputColumn { VarType::I64 => VariableValue::from(i64::from_u64(value)), VarType::U64 => VariableValue::from(value), VarType::F64 => VariableValue::from(f64::from_u64(value)), - VarType::Str => VariableValue::from(self.term_resolver.term_for_ord(value)), + VarType::Str => VariableValue::from(term_resolver.term_for_ord(value)), VarType::None => VariableValue::none(), } } /// Returns the index of the column: which docs have a value. pub(super) fn column_index(&self) -> &ColumnIndex { - &self.column.index + match &self.values { + InputValues::Ordinals { column, .. } => &column.index, + InputValues::PlainStr(plain_str_input) => { + plain_str_input.reader.column().column_index() + } + } + } +} + +/// Decodes the first value of docs from a plain-encoded `Str` column. +struct PlainStrInput { + reader: PlainValueReader, + /// The first values of the docs of the last block, concatenated. + block_values: String, + /// `block_value_ranges[doc_pos]` is the range of the value of `docs[doc_pos]` in + /// `block_values`, for the docs of the last block. `None` if the doc has no value. + block_value_ranges: Vec>>, + /// The value decoded by the last `value_for_doc` call. + single_value: String, +} + +impl PlainStrInput { + fn new(plain_str_column: &PlainStrColumn) -> PlainStrInput { + PlainStrInput { + reader: plain_str_column.reader(), + block_values: String::new(), + block_value_ranges: Vec::with_capacity(COLLECT_BLOCK_BUFFER_LEN), + single_value: String::new(), + } + } + + /// Appends the first value of `doc` to `output`. Returns false if `doc` has no value. + /// + /// Panics if the column cannot be read: `load_block` cannot return an error. + fn append_first_value(reader: &mut PlainValueReader, doc: DocId, output: &mut String) -> bool { + let value_range = reader.column().column_index().value_row_ids(doc); + if value_range.is_empty() { + return false; + } + reader + .append_str_value(value_range.start, output) + .expect("plain-encoded fast field is corrupted"); + true + } + + fn load_block(&mut self, docs: &[DocId]) { + self.block_values.clear(); + self.block_value_ranges.clear(); + // Docs are sorted: the blocks of the column are read in order. + for &doc in docs { + let start = self.block_values.len(); + if Self::append_first_value(&mut self.reader, doc, &mut self.block_values) { + self.block_value_ranges + .push(Some(start..self.block_values.len())); + } else { + self.block_value_ranges.push(None); + } + } + } + + #[inline] + fn value(&self, doc_pos: usize) -> Option<&str> { + let value_range = self.block_value_ranges[doc_pos].clone()?; + Some(&self.block_values[value_range]) + } + + fn value_for_doc(&mut self, doc: DocId) -> Option<&str> { + self.single_value.clear(); + if !Self::append_first_value(&mut self.reader, doc, &mut self.single_value) { + return None; + } + Some(self.single_value.as_str()) } } diff --git a/src/aggregation/value_source/jitexpr_value_source/mod.rs b/src/aggregation/value_source/jitexpr_value_source/mod.rs index 9cd736a18..53bd4a440 100644 --- a/src/aggregation/value_source/jitexpr_value_source/mod.rs +++ b/src/aggregation/value_source/jitexpr_value_source/mod.rs @@ -172,6 +172,55 @@ impl JitExprValueSourceProvider { } } +/// Name of the variable of the identity expression of `plain_str_value_source`. +/// +/// The variable is bound to a column handle directly, so its name does not matter. A constant +/// name lets all of the plain columns share the same compiled function. +const PLAIN_STR_IDENTITY_VARIABLE: &str = "value"; + +/// Returns a value source with the values of the plain-encoded `Str` column of `handle`. +/// +/// The source evaluates the identity jitexpr expression on the column. Its text results are +/// interned in the term dictionary of the source, like the results of any text expression: the +/// term ords are in first-seen order, and not sorted with the terms. +/// +/// A jitexpr source reads the first value of each doc only. Multivalued columns would silently +/// lose their other values, so they return an error. +pub(crate) fn plain_str_value_source( + reader: &SegmentReader, + handle: &DynamicColumnHandle, +) -> crate::Result> { + let input = InputColumn::open(handle)?; + if input.var_type() != VarType::Str { + return Err(TantivyError::InternalError( + "a plain column source requires a text column".to_string(), + )); + } + if input.column_index().get_cardinality().is_multivalue() { + return Err(TantivyError::InvalidArgument( + "aggregations are not supported on multivalued plain-encoded text fast fields" + .to_string(), + )); + } + let identity = UntypedExpr::variable(PLAIN_STR_IDENTITY_VARIABLE); + let variable_types: HashMap<&str, VarType> = + HashMap::from([(PLAIN_STR_IDENTITY_VARIABLE, VarType::Str)]); + let compiled_fn: Arc = reader + .index() + .expr_compilation_cache() + .compile(&identity, &variable_types) + .map_err(|compile_error| { + TantivyError::InternalError(format!( + "the compilation of the identity expression failed: {compile_error}" + )) + })?; + Ok(Box::new(JitExprValueSource::new( + compiled_fn.context(), + ColumnType::Str, + vec![input], + ))) +} + /// The source of a jitexpr expression on a segment, returned by /// `JitExprValueSourceProvider::open_segment`. pub(crate) enum JitExprSegmentSource { @@ -902,4 +951,168 @@ mod tests { } assert_eq!(expected["tripled_stats"]["count"], 398); } + + /// Two segments. "label" is a plain-encoded copy of the dictionary encoded "label_dict". + /// "tags" is a multivalued plain-encoded field. + fn create_plain_label_index() -> Index { + use crate::schema::{StrColumnCodec, TextOptions}; + + let plain_text_options = TextOptions::default().set_fast_codec(StrColumnCodec::Plain); + let mut schema_builder = Schema::builder(); + let label = schema_builder.add_text_field("label", plain_text_options.clone()); + let label_dict = schema_builder.add_text_field("label_dict", STRING | FAST); + let tags = schema_builder.add_text_field("tags", plain_text_options); + let score = schema_builder.add_u64_field("score", FAST); + let index = Index::create_in_ram(schema_builder.build()); + let mut writer = index.writer_for_tests().unwrap(); + for (label_value, score_value) in [("b", 1u64), ("a", 2), ("b", 3), ("c", 4)] { + writer + .add_document(doc!( + label => label_value, + label_dict => label_value, + score => score_value, + tags => "x", + tags => label_value, + )) + .unwrap(); + } + writer.add_document(doc!(score => 100u64)).unwrap(); + writer.commit().unwrap(); + for (label_value, score_value) in [("c", 5u64), ("a", 6), ("b", 7)] { + writer + .add_document( + doc!(label => label_value, label_dict => label_value, score => score_value), + ) + .unwrap(); + } + writer.commit().unwrap(); + index + } + + #[test] + fn test_terms_over_plain_text_field_matches_dictionary() { + let index = create_plain_label_index(); + for terms_req in [ + json!({ "field": "FIELD" }), + json!({ "field": "FIELD", "size": 2 }), + json!({ "field": "FIELD", "order": { "score_sum": "desc" } }), + ] { + let mut results: Vec = Vec::new(); + for field in ["label", "label_dict"] { + let mut terms_req = terms_req.clone(); + terms_req["field"] = json!(field); + let result = run_agg( + &index, + &[], + json!({ + "t": { + "terms": terms_req, + "aggs": { "score_sum": { "sum": { "field": "score" } } } + } + }), + ); + results.push(result); + } + assert_eq!(results[0], results[1], "{terms_req}"); + } + let result = run_agg( + &index, + &[], + json!({ "t": { "terms": { "field": "label" } } }), + ); + assert_eq!( + sorted_bucket_counts(&result, "t"), + vec![ + ("\"a\"".to_string(), 2), + ("\"b\"".to_string(), 3), + ("\"c\"".to_string(), 2) + ] + ); + } + + #[test] + fn test_cardinality_over_plain_text_field() { + let index = create_plain_label_index(); + let result = run_agg( + &index, + &[], + json!({ + "plain": { "cardinality": { "field": "label" } }, + "dict": { "cardinality": { "field": "label_dict" } } + }), + ); + assert_eq!(result["plain"]["value"], 3.0); + assert_eq!(result["plain"], result["dict"]); + } + + #[test] + fn test_jitexpr_field_over_plain_text_input() { + let index = create_plain_label_index(); + let result = run_agg( + &index, + &[ + ("#upper", "(UPPER label)"), + ("#upper_dict", "(UPPER label_dict)"), + ], + json!({ + "plain": { "terms": { "field": "#upper" } }, + "dict": { "terms": { "field": "#upper_dict" } } + }), + ); + assert_eq!( + sorted_bucket_counts(&result, "plain"), + vec![ + ("\"A\"".to_string(), 2), + ("\"B\"".to_string(), 3), + ("\"C\"".to_string(), 2) + ] + ); + assert_eq!(result["plain"], result["dict"]); + } + + #[test] + fn test_unsupported_aggregations_on_plain_text_field() { + let index = create_plain_label_index(); + for (aggs, expected_message) in [ + ( + json!({ "t": { "terms": { "field": "label", "order": { "_key": "asc" } } } }), + "ordered by `_key`", + ), + ( + json!({ "t": { "terms": { "field": "label", "min_doc_count": 0 } } }), + "`min_doc_count: 0`", + ), + ( + json!({ "t": { "terms": { "field": "label", "include": "a.*" } } }), + "`include` / `exclude`", + ), + ( + json!({ "t": { "terms": { "field": "label", "exclude": ["a"] } } }), + "`include` / `exclude`", + ), + ( + json!({ "t": { "terms": { "field": "label", "missing": "NONE" } } }), + "`missing`", + ), + ( + json!({ "t": { "terms": { "field": "tags" } } }), + "multivalued plain-encoded", + ), + ( + json!({ "t": { "cardinality": { "field": "tags" } } }), + "multivalued plain-encoded", + ), + ( + json!({ "t": { "multi_terms": { "terms": [{ "field": "label" }, { "field": "score" }] } } }), + "plain", + ), + ( + json!({ "t": { "composite": { "size": 10, "sources": [{ "l": { "terms": { "field": "label" } } }] } } }), + "plain", + ), + ] { + let err = try_run_agg(&index, &[], aggs.clone()).unwrap_err(); + assert!(err.to_string().contains(expected_message), "{aggs}: {err}"); + } + } } diff --git a/src/query/doc_predicate_query/jitexpr_predicate.rs b/src/query/doc_predicate_query/jitexpr_predicate.rs index 78cb61d11..75235ed8e 100644 --- a/src/query/doc_predicate_query/jitexpr_predicate.rs +++ b/src/query/doc_predicate_query/jitexpr_predicate.rs @@ -209,6 +209,52 @@ mod tests { .into() } + /// "label" is plain-encoded, "label_dict" is its dictionary encoded copy. + fn create_plain_index() -> Index { + use crate::schema::{StrColumnCodec, TextOptions}; + + let mut schema_builder = Schema::builder(); + let label = schema_builder.add_text_field( + "label", + TextOptions::default().set_fast_codec(StrColumnCodec::Plain), + ); + let label_dict = schema_builder.add_text_field("label_dict", STRING | FAST); + let index = Index::create_in_ram(schema_builder.build()); + let mut writer = index.writer_for_tests().unwrap(); + for label_value in ["apple", "banana", "avocado", "cherry"] { + writer + .add_document(doc!(label => label_value, label_dict => label_value)) + .unwrap(); + } + writer.add_document(doc!()).unwrap(); + writer.commit().unwrap(); + writer + .add_document(doc!(label => "apricot", label_dict => "apricot")) + .unwrap(); + writer.commit().unwrap(); + index + } + + #[test] + fn test_predicate_over_plain_text_field() { + let index = create_plain_index(); + let searcher = index.reader().unwrap().searcher(); + for (expression, expected_count) in [ + (r#"(EQ label "banana")"#, 1), + (r#"(EQ (UPPER label) "CHERRY")"#, 1), + (r#"(REGEXP_LIKE label "^a")"#, 3), + (r#"(EQ label "missing")"#, 0), + ("(IS_NULL label)", 1), + ("(IS_NOT_NULL label)", 5), + ] { + let count = searcher.search(&query(expression), &Count).unwrap(); + assert_eq!(count, expected_count, "{expression}"); + let dict_expression = expression.replace("label", "label_dict"); + let dict_count = searcher.search(&query(&dict_expression), &Count).unwrap(); + assert_eq!(count, dict_count, "{expression}"); + } + } + #[test] fn test_constructor_requires_boolean_expression() { let expression = jitexpr::ast::deserialize("(ADD number 1u64)").unwrap();