Support plain-encoded text fields in JitExpr and aggregations

JitExpr inputs can be plain-encoded text columns, for predicates and
calculated fields.

Terms and cardinality aggregations read plain text columns through the
identity jitexpr expression, which interns the values. Options relying on
sorted term ordinals, missing values or multivalued plain columns return
an error.
This commit is contained in:
Paul Masurel
2026-10-13 08:36:27 +02:00
parent 9c5ce3baad
commit 6e79b1ace0
5 changed files with 494 additions and 47 deletions
+60 -2
View File
@@ -2,7 +2,9 @@
use std::io;
use columnar::{BytesColumn, Column, ColumnType, DynamicColumn, DynamicColumnHandle, StrColumn};
use columnar::{
BytesColumn, Column, ColumnType, DynamicColumn, DynamicColumnHandle, StrColumn, StrColumnCodec,
};
use crate::aggregation::value_source::ValueSource;
use crate::aggregation::{f64_to_fastfield_u64, Key, ValueSourceRegistry};
@@ -144,6 +146,20 @@ pub(crate) fn get_all_value_sources(
Ok(sources)
}
#[cfg(feature = "jitexpr")]
use crate::aggregation::jitexpr_value_source::plain_str_value_source;
/// Aggregations on plain-encoded text columns require the `jitexpr` feature.
#[cfg(not(feature = "jitexpr"))]
fn plain_str_value_source(
_reader: &SegmentReader,
_handle: &DynamicColumnHandle,
) -> crate::Result<Box<dyn ValueSource>> {
Err(crate::TantivyError::InvalidArgument(
"aggregations on plain-encoded text fast fields require the `jitexpr` feature".to_string(),
))
}
/// Builds a value source without any value, with the given type.
///
/// `num_rows` is the number of rows of the column (the `max_doc` of the segment).
@@ -183,7 +199,7 @@ fn open_physical_column_value_sources(
continue;
}
}
if let Some(column) = open_physical_column_value_source(handle)? {
if let Some(column) = open_physical_column_value_source(reader, handle)? {
sources.push(column);
if first_only {
break;
@@ -196,10 +212,22 @@ fn open_physical_column_value_sources(
/// Opens a column as a value source.
///
/// Returns `None` if the column cannot be read as `u64`.
///
/// A plain-encoded text column has no term ordinals: it is exposed through a jitexpr source
/// interning its values, see `plain_str_value_source`.
fn open_physical_column_value_source(
reader: &SegmentReader,
column_handle: DynamicColumnHandle,
) -> crate::Result<Option<Box<dyn ValueSource>>> {
let column_type = column_handle.column_type();
if column_handle.str_codec() == Some(StrColumnCodec::Plain) {
if column_type != ColumnType::Str {
return Err(crate::TantivyError::InvalidArgument(format!(
"aggregations are not supported on plain-encoded {column_type} fast fields"
)));
}
return plain_str_value_source(reader, &column_handle).map(Some);
}
if column_type == ColumnType::Str {
let DynamicColumn::Str(str_column) = column_handle.open()? else {
return Err(crate::TantivyError::InternalError(
@@ -213,3 +241,33 @@ fn open_physical_column_value_source(
};
Ok(Some(Box::new((column, column_type))))
}
#[cfg(all(test, not(feature = "jitexpr")))]
mod tests {
use serde_json::json;
use crate::aggregation::agg_req::Aggregations;
use crate::aggregation::AggregationCollector;
use crate::query::AllQuery;
use crate::schema::{Schema, StrColumnCodec, TextOptions};
use crate::Index;
#[test]
fn test_aggregation_on_plain_text_field_requires_jitexpr() {
let mut schema_builder = Schema::builder();
let label = schema_builder.add_text_field(
"label",
TextOptions::default().set_fast_codec(StrColumnCodec::Plain),
);
let index = Index::create_in_ram(schema_builder.build());
let mut writer = index.writer_for_tests().unwrap();
writer.add_document(doc!(label => "a")).unwrap();
writer.commit().unwrap();
let aggs: Aggregations =
serde_json::from_value(json!({ "t": { "terms": { "field": "label" } } })).unwrap();
let collector = AggregationCollector::from_aggs(aggs, Default::default());
let searcher = index.reader().unwrap().searcher();
let err = searcher.search(&AllQuery, &collector).unwrap_err();
assert!(err.to_string().contains("`jitexpr` feature"), "{err}");
}
}
+7 -6
View File
@@ -292,7 +292,8 @@ fn missing_value_for_source(
Some(column) => column.max_value(),
None if column_type == ColumnType::Str => {
return Err(crate::TantivyError::InvalidArgument(format!(
"`missing` is not supported for the computed text value source `{field_name}`"
"`missing` is not supported for the computed or plain-encoded text field \
`{field_name}`"
)));
}
None => 0,
@@ -311,7 +312,7 @@ fn require_physical_column(
) -> crate::Result<Column<u64>> {
source.as_physical_column().cloned().ok_or_else(|| {
crate::TantivyError::InvalidArgument(format!(
"{agg_kind} does not support the computed value source `{field_name}`"
"{agg_kind} does not support the computed or plain-encoded field `{field_name}`"
))
})
}
@@ -326,7 +327,7 @@ fn reject_registered_source(
) -> crate::Result<()> {
if value_sources.get(field_name).is_some() {
return Err(crate::TantivyError::InvalidArgument(format!(
"{agg_kind} does not support the computed value source `{field_name}`"
"{agg_kind} does not support the computed or plain-encoded field `{field_name}`"
)));
}
Ok(())
@@ -1003,7 +1004,7 @@ fn build_terms_or_cardinality_nodes(
let Some(str_col) = accessor.as_physical_str_column() else {
return Err(crate::TantivyError::InvalidArgument(format!(
"terms aggregation with `include` / `exclude` requires a physical \
text field, but `{field_name}` is a computed value source"
text field, but `{field_name}` is a computed or plain-encoded field"
)));
};
allowed_term_ids = build_allowed_term_ids_for_str(
@@ -1026,14 +1027,14 @@ fn build_terms_or_cardinality_nodes(
{
return Err(crate::TantivyError::InvalidArgument(format!(
"terms aggregation ordered by `_key` is not supported on the computed \
value source `{field_name}`"
or plain-encoded text field `{field_name}`"
)));
}
// `min_doc_count: 0` requires listing all of the possible values.
if req_internal.min_doc_count == 0 {
return Err(crate::TantivyError::InvalidArgument(format!(
"terms aggregation with `min_doc_count: 0` is not supported on the \
computed value source `{field_name}`"
computed or plain-encoded field `{field_name}`"
)));
}
}
@@ -1,7 +1,10 @@
//! The fast-field columns bound to the inputs of a compiled jitexpr expression.
use std::ops::Range;
use columnar::{
Column, ColumnIndex, DynamicColumn, DynamicColumnHandle, MonotonicallyMappableToU64,
PlainStrColumn, PlainValueReader,
};
use jitexpr::types::{VarType, VariableValue};
@@ -12,12 +15,21 @@ use crate::{DocId, TantivyError, COLLECT_BLOCK_BUFFER_LEN};
/// A fast-field column bound to an input of the compiled expression.
pub(super) struct InputColumn {
var_type: VarType,
/// Values in their monotonic `u64` mapping. Term ords for a `Str` input.
column: Column<u64>,
/// `first_values[i]` is the first value of `docs[i]`, for the docs of the last block.
first_values: Vec<Option<u64>>,
/// The terms of `first_values`. Not `TermResolver::None` if and only if `var_type` is `Str`.
term_resolver: TermResolver,
values: InputValues,
}
enum InputValues {
/// Values in their monotonic `u64` mapping. Term ords for a dictionary encoded `Str` input.
Ordinals {
column: Column<u64>,
/// `first_values[i]` is the first value of `docs[i]`, for the docs of the last block.
first_values: Vec<Option<u64>>,
/// The terms of `first_values`. Not `TermResolver::None` if and only if `var_type` is
/// `Str`.
term_resolver: TermResolver,
},
/// A plain-encoded `Str` input: the values are decoded from the column.
PlainStr(PlainStrInput),
}
impl InputColumn {
@@ -30,31 +42,39 @@ impl InputColumn {
)));
};
if var_type == VarType::Str {
let DynamicColumn::Str(str_column) = handle.open()? else {
return Err(TantivyError::InternalError(
"a text column could not be opened as a text column".to_string(),
));
let values = match handle.open()? {
DynamicColumn::Str(str_column) => {
let (term_dictionary, term_ords) = str_column.into_parts();
InputValues::Ordinals {
column: term_ords,
first_values: Vec::with_capacity(COLLECT_BLOCK_BUFFER_LEN),
term_resolver: TermResolver::for_dictionary(term_dictionary)?,
}
}
DynamicColumn::PlainStr(plain_str_column) => {
InputValues::PlainStr(PlainStrInput::new(&plain_str_column))
}
_ => {
return Err(TantivyError::InternalError(
"a text column could not be opened as a text column".to_string(),
));
}
};
let (term_dictionary, term_ords) = str_column.into_parts();
Ok(InputColumn {
var_type,
column: term_ords,
first_values: Vec::with_capacity(COLLECT_BLOCK_BUFFER_LEN),
term_resolver: TermResolver::for_dictionary(term_dictionary)?,
})
} else {
let column = handle.open_u64_lenient()?.ok_or_else(|| {
TantivyError::InternalError(format!(
"a column of type {column_type} could not be opened as u64"
))
})?;
Ok(InputColumn {
var_type,
return Ok(InputColumn { var_type, values });
}
let column = handle.open_u64_lenient()?.ok_or_else(|| {
TantivyError::InternalError(format!(
"a column of type {column_type} could not be opened as u64"
))
})?;
Ok(InputColumn {
var_type,
values: InputValues::Ordinals {
column,
first_values: Vec::with_capacity(COLLECT_BLOCK_BUFFER_LEN),
term_resolver: TermResolver::None,
})
}
},
})
}
pub(super) fn var_type(&self) -> VarType {
@@ -63,19 +83,43 @@ impl InputColumn {
/// Loads the first value of each doc of `docs`.
///
/// Panics if the dictionary of a text column is corrupted.
/// Panics if the dictionary of a text column is corrupted, or if a plain column cannot be
/// read.
pub(super) fn load_block(&mut self, docs: &[DocId]) {
// `first_vals` does not write the entries of the docs without a value: we reset them.
self.first_values.clear();
self.first_values.resize(docs.len(), None);
self.column.first_vals(docs, &mut self.first_values);
self.term_resolver.load(&self.first_values);
match &mut self.values {
InputValues::Ordinals {
column,
first_values,
term_resolver,
} => {
// `first_vals` does not write the entries of the docs without a value: we reset
// them.
first_values.clear();
first_values.resize(docs.len(), None);
column.first_vals(docs, first_values);
term_resolver.load(first_values);
}
InputValues::PlainStr(plain_str_input) => plain_str_input.load_block(docs),
}
}
/// Returns the value of `docs[doc_pos]`, for the `docs` of the last `load_block` call.
#[inline]
pub(super) fn value(&self, doc_pos: usize) -> VariableValue<'_> {
let Some(value) = self.first_values[doc_pos] else {
let (first_values, term_resolver) = match &self.values {
InputValues::Ordinals {
first_values,
term_resolver,
..
} => (first_values, term_resolver),
InputValues::PlainStr(plain_str_input) => {
return plain_str_input
.value(doc_pos)
.map(VariableValue::from)
.unwrap_or(VariableValue::none());
}
};
let Some(value) = first_values[doc_pos] else {
return VariableValue::none();
};
match self.var_type {
@@ -83,7 +127,7 @@ impl InputColumn {
VarType::I64 => VariableValue::from(i64::from_u64(value)),
VarType::U64 => VariableValue::from(value),
VarType::F64 => VariableValue::from(f64::from_u64(value)),
VarType::Str => VariableValue::from(self.term_resolver.term(doc_pos, value)),
VarType::Str => VariableValue::from(term_resolver.term(doc_pos, value)),
VarType::None => VariableValue::none(),
}
}
@@ -93,10 +137,24 @@ impl InputColumn {
/// Invalidates the last `load_block` call: `value` must not be called before the next
/// `load_block`.
///
/// Panics if the dictionary of a text column is corrupted.
/// Panics if the dictionary of a text column is corrupted, or if a plain column cannot be
/// read.
#[inline]
pub(super) fn value_for_doc(&mut self, doc: DocId) -> VariableValue<'_> {
let Some(value) = self.column.first(doc) else {
let (column, term_resolver) = match &mut self.values {
InputValues::Ordinals {
column,
term_resolver,
..
} => (column, term_resolver),
InputValues::PlainStr(plain_str_input) => {
return plain_str_input
.value_for_doc(doc)
.map(VariableValue::from)
.unwrap_or(VariableValue::none());
}
};
let Some(value) = column.first(doc) else {
return VariableValue::none();
};
match self.var_type {
@@ -104,13 +162,84 @@ impl InputColumn {
VarType::I64 => VariableValue::from(i64::from_u64(value)),
VarType::U64 => VariableValue::from(value),
VarType::F64 => VariableValue::from(f64::from_u64(value)),
VarType::Str => VariableValue::from(self.term_resolver.term_for_ord(value)),
VarType::Str => VariableValue::from(term_resolver.term_for_ord(value)),
VarType::None => VariableValue::none(),
}
}
/// Returns the index of the column: which docs have a value.
pub(super) fn column_index(&self) -> &ColumnIndex {
&self.column.index
match &self.values {
InputValues::Ordinals { column, .. } => &column.index,
InputValues::PlainStr(plain_str_input) => {
plain_str_input.reader.column().column_index()
}
}
}
}
/// Decodes the first value of docs from a plain-encoded `Str` column.
struct PlainStrInput {
reader: PlainValueReader,
/// The first values of the docs of the last block, concatenated.
block_values: String,
/// `block_value_ranges[doc_pos]` is the range of the value of `docs[doc_pos]` in
/// `block_values`, for the docs of the last block. `None` if the doc has no value.
block_value_ranges: Vec<Option<Range<usize>>>,
/// The value decoded by the last `value_for_doc` call.
single_value: String,
}
impl PlainStrInput {
fn new(plain_str_column: &PlainStrColumn) -> PlainStrInput {
PlainStrInput {
reader: plain_str_column.reader(),
block_values: String::new(),
block_value_ranges: Vec::with_capacity(COLLECT_BLOCK_BUFFER_LEN),
single_value: String::new(),
}
}
/// Appends the first value of `doc` to `output`. Returns false if `doc` has no value.
///
/// Panics if the column cannot be read: `load_block` cannot return an error.
fn append_first_value(reader: &mut PlainValueReader, doc: DocId, output: &mut String) -> bool {
let value_range = reader.column().column_index().value_row_ids(doc);
if value_range.is_empty() {
return false;
}
reader
.append_str_value(value_range.start, output)
.expect("plain-encoded fast field is corrupted");
true
}
fn load_block(&mut self, docs: &[DocId]) {
self.block_values.clear();
self.block_value_ranges.clear();
// Docs are sorted: the blocks of the column are read in order.
for &doc in docs {
let start = self.block_values.len();
if Self::append_first_value(&mut self.reader, doc, &mut self.block_values) {
self.block_value_ranges
.push(Some(start..self.block_values.len()));
} else {
self.block_value_ranges.push(None);
}
}
}
#[inline]
fn value(&self, doc_pos: usize) -> Option<&str> {
let value_range = self.block_value_ranges[doc_pos].clone()?;
Some(&self.block_values[value_range])
}
fn value_for_doc(&mut self, doc: DocId) -> Option<&str> {
self.single_value.clear();
if !Self::append_first_value(&mut self.reader, doc, &mut self.single_value) {
return None;
}
Some(self.single_value.as_str())
}
}
@@ -172,6 +172,55 @@ impl JitExprValueSourceProvider {
}
}
/// Name of the variable of the identity expression of `plain_str_value_source`.
///
/// The variable is bound to a column handle directly, so its name does not matter. A constant
/// name lets all of the plain columns share the same compiled function.
const PLAIN_STR_IDENTITY_VARIABLE: &str = "value";
/// Returns a value source with the values of the plain-encoded `Str` column of `handle`.
///
/// The source evaluates the identity jitexpr expression on the column. Its text results are
/// interned in the term dictionary of the source, like the results of any text expression: the
/// term ords are in first-seen order, and not sorted with the terms.
///
/// A jitexpr source reads the first value of each doc only. Multivalued columns would silently
/// lose their other values, so they return an error.
pub(crate) fn plain_str_value_source(
reader: &SegmentReader,
handle: &DynamicColumnHandle,
) -> crate::Result<Box<dyn ValueSource>> {
let input = InputColumn::open(handle)?;
if input.var_type() != VarType::Str {
return Err(TantivyError::InternalError(
"a plain column source requires a text column".to_string(),
));
}
if input.column_index().get_cardinality().is_multivalue() {
return Err(TantivyError::InvalidArgument(
"aggregations are not supported on multivalued plain-encoded text fast fields"
.to_string(),
));
}
let identity = UntypedExpr::variable(PLAIN_STR_IDENTITY_VARIABLE);
let variable_types: HashMap<&str, VarType> =
HashMap::from([(PLAIN_STR_IDENTITY_VARIABLE, VarType::Str)]);
let compiled_fn: Arc<CompiledFn> = reader
.index()
.expr_compilation_cache()
.compile(&identity, &variable_types)
.map_err(|compile_error| {
TantivyError::InternalError(format!(
"the compilation of the identity expression failed: {compile_error}"
))
})?;
Ok(Box::new(JitExprValueSource::new(
compiled_fn.context(),
ColumnType::Str,
vec![input],
)))
}
/// The source of a jitexpr expression on a segment, returned by
/// `JitExprValueSourceProvider::open_segment`.
pub(crate) enum JitExprSegmentSource {
@@ -902,4 +951,168 @@ mod tests {
}
assert_eq!(expected["tripled_stats"]["count"], 398);
}
/// Two segments. "label" is a plain-encoded copy of the dictionary encoded "label_dict".
/// "tags" is a multivalued plain-encoded field.
fn create_plain_label_index() -> Index {
use crate::schema::{StrColumnCodec, TextOptions};
let plain_text_options = TextOptions::default().set_fast_codec(StrColumnCodec::Plain);
let mut schema_builder = Schema::builder();
let label = schema_builder.add_text_field("label", plain_text_options.clone());
let label_dict = schema_builder.add_text_field("label_dict", STRING | FAST);
let tags = schema_builder.add_text_field("tags", plain_text_options);
let score = schema_builder.add_u64_field("score", FAST);
let index = Index::create_in_ram(schema_builder.build());
let mut writer = index.writer_for_tests().unwrap();
for (label_value, score_value) in [("b", 1u64), ("a", 2), ("b", 3), ("c", 4)] {
writer
.add_document(doc!(
label => label_value,
label_dict => label_value,
score => score_value,
tags => "x",
tags => label_value,
))
.unwrap();
}
writer.add_document(doc!(score => 100u64)).unwrap();
writer.commit().unwrap();
for (label_value, score_value) in [("c", 5u64), ("a", 6), ("b", 7)] {
writer
.add_document(
doc!(label => label_value, label_dict => label_value, score => score_value),
)
.unwrap();
}
writer.commit().unwrap();
index
}
#[test]
fn test_terms_over_plain_text_field_matches_dictionary() {
let index = create_plain_label_index();
for terms_req in [
json!({ "field": "FIELD" }),
json!({ "field": "FIELD", "size": 2 }),
json!({ "field": "FIELD", "order": { "score_sum": "desc" } }),
] {
let mut results: Vec<Value> = Vec::new();
for field in ["label", "label_dict"] {
let mut terms_req = terms_req.clone();
terms_req["field"] = json!(field);
let result = run_agg(
&index,
&[],
json!({
"t": {
"terms": terms_req,
"aggs": { "score_sum": { "sum": { "field": "score" } } }
}
}),
);
results.push(result);
}
assert_eq!(results[0], results[1], "{terms_req}");
}
let result = run_agg(
&index,
&[],
json!({ "t": { "terms": { "field": "label" } } }),
);
assert_eq!(
sorted_bucket_counts(&result, "t"),
vec![
("\"a\"".to_string(), 2),
("\"b\"".to_string(), 3),
("\"c\"".to_string(), 2)
]
);
}
#[test]
fn test_cardinality_over_plain_text_field() {
let index = create_plain_label_index();
let result = run_agg(
&index,
&[],
json!({
"plain": { "cardinality": { "field": "label" } },
"dict": { "cardinality": { "field": "label_dict" } }
}),
);
assert_eq!(result["plain"]["value"], 3.0);
assert_eq!(result["plain"], result["dict"]);
}
#[test]
fn test_jitexpr_field_over_plain_text_input() {
let index = create_plain_label_index();
let result = run_agg(
&index,
&[
("#upper", "(UPPER label)"),
("#upper_dict", "(UPPER label_dict)"),
],
json!({
"plain": { "terms": { "field": "#upper" } },
"dict": { "terms": { "field": "#upper_dict" } }
}),
);
assert_eq!(
sorted_bucket_counts(&result, "plain"),
vec![
("\"A\"".to_string(), 2),
("\"B\"".to_string(), 3),
("\"C\"".to_string(), 2)
]
);
assert_eq!(result["plain"], result["dict"]);
}
#[test]
fn test_unsupported_aggregations_on_plain_text_field() {
let index = create_plain_label_index();
for (aggs, expected_message) in [
(
json!({ "t": { "terms": { "field": "label", "order": { "_key": "asc" } } } }),
"ordered by `_key`",
),
(
json!({ "t": { "terms": { "field": "label", "min_doc_count": 0 } } }),
"`min_doc_count: 0`",
),
(
json!({ "t": { "terms": { "field": "label", "include": "a.*" } } }),
"`include` / `exclude`",
),
(
json!({ "t": { "terms": { "field": "label", "exclude": ["a"] } } }),
"`include` / `exclude`",
),
(
json!({ "t": { "terms": { "field": "label", "missing": "NONE" } } }),
"`missing`",
),
(
json!({ "t": { "terms": { "field": "tags" } } }),
"multivalued plain-encoded",
),
(
json!({ "t": { "cardinality": { "field": "tags" } } }),
"multivalued plain-encoded",
),
(
json!({ "t": { "multi_terms": { "terms": [{ "field": "label" }, { "field": "score" }] } } }),
"plain",
),
(
json!({ "t": { "composite": { "size": 10, "sources": [{ "l": { "terms": { "field": "label" } } }] } } }),
"plain",
),
] {
let err = try_run_agg(&index, &[], aggs.clone()).unwrap_err();
assert!(err.to_string().contains(expected_message), "{aggs}: {err}");
}
}
}
@@ -209,6 +209,52 @@ mod tests {
.into()
}
/// "label" is plain-encoded, "label_dict" is its dictionary encoded copy.
fn create_plain_index() -> Index {
use crate::schema::{StrColumnCodec, TextOptions};
let mut schema_builder = Schema::builder();
let label = schema_builder.add_text_field(
"label",
TextOptions::default().set_fast_codec(StrColumnCodec::Plain),
);
let label_dict = schema_builder.add_text_field("label_dict", STRING | FAST);
let index = Index::create_in_ram(schema_builder.build());
let mut writer = index.writer_for_tests().unwrap();
for label_value in ["apple", "banana", "avocado", "cherry"] {
writer
.add_document(doc!(label => label_value, label_dict => label_value))
.unwrap();
}
writer.add_document(doc!()).unwrap();
writer.commit().unwrap();
writer
.add_document(doc!(label => "apricot", label_dict => "apricot"))
.unwrap();
writer.commit().unwrap();
index
}
#[test]
fn test_predicate_over_plain_text_field() {
let index = create_plain_index();
let searcher = index.reader().unwrap().searcher();
for (expression, expected_count) in [
(r#"(EQ label "banana")"#, 1),
(r#"(EQ (UPPER label) "CHERRY")"#, 1),
(r#"(REGEXP_LIKE label "^a")"#, 3),
(r#"(EQ label "missing")"#, 0),
("(IS_NULL label)", 1),
("(IS_NOT_NULL label)", 5),
] {
let count = searcher.search(&query(expression), &Count).unwrap();
assert_eq!(count, expected_count, "{expression}");
let dict_expression = expression.replace("label", "label_dict");
let dict_count = searcher.search(&query(&dict_expression), &Count).unwrap();
assert_eq!(count, dict_count, "{expression}");
}
}
#[test]
fn test_constructor_requires_boolean_expression() {
let expression = jitexpr::ast::deserialize("(ADD number 1u64)").unwrap();