From 51d3000ce63384b2543060dae26517aed19a7d0b Mon Sep 17 00:00:00 2001 From: DavIvek Date: Thu, 18 Jun 2026 15:38:28 +0200 Subject: [PATCH] fix(query-parser): honor phrase prefix and slop on JSON fields Phrase prefix (`"..."*`) and slop (`"..."~N`) were silently dropped on JSON fields. `generate_literals_for_json_object` hard-coded `slop: 0` and `prefix: false`, so a query like `data.name:"foo bar"*` degraded to an exact phrase, even though the underlying `PhrasePrefixQuery` already supports JSON-path terms. Thread `slop`/`prefix` through the JSON branch, exactly as the `Str` branch already does, and add the "phrase prefix requires at least two terms" guard for the single-token case (mirroring `generate_literals_for_str`). Tests added as JSON analogues of the existing text-field tests: - test_phrase_prefix_on_json_field - test_phrase_prefix_too_short_on_json_field - test_phrase_slop_on_json_field This aligns JSON fields with the phrase `~`/`*` behavior already documented for `QueryParser`. --- CHANGELOG.md | 1 + src/query/query_parser/query_parser.rs | 51 +++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eedb2d8b4..0864e556a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ Tantivy 0.26 (Unreleased) - Fix integer overflow in segment sorting and merge policy truncation [#2846](https://github.com/quickwit-oss/tantivy/pull/2846)(@anaslimem) - Fix merging of intermediate aggregation results [#2719](https://github.com/quickwit-oss/tantivy/pull/2719)(@PSeitz) - Fix deduplicate doc counts in term aggregation for multi-valued fields [#2854](https://github.com/quickwit-oss/tantivy/pull/2854)(@nuri-yoo) +- Honor phrase prefix (`"..."*`) and slop (`"..."~N`) on JSON fields; previously both were silently dropped, degrading the query to an exact phrase [#2966](https://github.com/quickwit-oss/tantivy/pull/2966)(@DavIvek) ## Features/Improvements - **Aggregation** diff --git a/src/query/query_parser/query_parser.rs b/src/query/query_parser/query_parser.rs index 4fb268a31..959c85588 100644 --- a/src/query/query_parser/query_parser.rs +++ b/src/query/query_parser/query_parser.rs @@ -611,6 +611,8 @@ impl QueryParser { field, json_path, phrase, + slop, + prefix, &self.tokenizer_manager, json_options, ), @@ -1008,11 +1010,14 @@ fn generate_literals_for_str( })) } +#[expect(clippy::too_many_arguments)] fn generate_literals_for_json_object( field_name: &str, field: Field, json_path: &str, phrase: &str, + slop: u32, + prefix: bool, tokenizer_manager: &TokenizerManager, json_options: &JsonObjectOptions, ) -> Result, QueryParserError> { @@ -1049,6 +1054,12 @@ fn generate_literals_for_json_object( }); if positions_and_terms.len() <= 1 { + if prefix { + return Err(QueryParserError::PhrasePrefixRequiresAtLeastTwoTerms { + phrase: phrase.to_owned(), + tokenizer: text_options.tokenizer().to_owned(), + }); + } for (_, term) in positions_and_terms { logical_literals.push(LogicalLiteral::Term(term)); } @@ -1061,8 +1072,8 @@ fn generate_literals_for_json_object( } logical_literals.push(LogicalLiteral::Phrase { terms: positions_and_terms, - slop: 0, - prefix: false, + slop, + prefix, }); Ok(logical_literals) } @@ -1977,6 +1988,42 @@ mod test { ); } + #[test] + pub fn test_phrase_prefix_on_json_field() { + let query_parser = make_query_parser(); + let query = query_parser + .parse_query("json.attr:\"big bad wo\"*") + .unwrap(); + assert_eq!( + format!("{query:?}"), + "PhrasePrefixQuery { field: Field(14), phrase_terms: [(0, Term(field=14, type=Json, \ + path=attr, type=Str, \"big\")), (1, Term(field=14, type=Json, path=attr, type=Str, \ + \"bad\"))], prefix: (2, Term(field=14, type=Json, path=attr, type=Str, \"wo\")), \ + max_expansions: 50 }" + ); + } + + #[test] + pub fn test_phrase_prefix_too_short_on_json_field() { + let err = parse_query_to_logical_ast("json.attr:\"wo\"*", true).unwrap_err(); + assert_eq!( + err, + QueryParserError::PhrasePrefixRequiresAtLeastTwoTerms { + phrase: "wo".to_owned(), + tokenizer: "default".to_owned() + } + ); + } + + #[test] + pub fn test_phrase_slop_on_json_field() { + test_parse_query_to_logical_ast_helper( + "json.attr:\"a b\"~2", + r#""[(0, Term(field=14, type=Json, path=attr, type=Str, "a")), (1, Term(field=14, type=Json, path=attr, type=Str, "b"))]"~2"#, + false, + ); + } + #[test] pub fn test_term_set_query() { test_parse_query_to_logical_ast_helper(