diff --git a/columnar/Cargo.toml b/columnar/Cargo.toml index 10b49a24d..2e7e55fc7 100644 --- a/columnar/Cargo.toml +++ b/columnar/Cargo.toml @@ -16,7 +16,7 @@ stacker = { version= "0.7", path = "../stacker", package="tantivy-stacker"} sstable = { version= "0.7", path = "../sstable", package = "tantivy-sstable" } common = { version= "0.11", path = "../common", package = "tantivy-common" } tantivy-bitpacker = { version= "0.10", path = "../bitpacker/" } -serde = "1.0.152" +serde = { version = "1.0.152", features = ["derive"] } downcast-rs = "2.0.1" [dev-dependencies] diff --git a/columnar/src/lib.rs b/columnar/src/lib.rs index 2db89ef28..869f75bb1 100644 --- a/columnar/src/lib.rs +++ b/columnar/src/lib.rs @@ -32,6 +32,7 @@ mod columnar; mod dictionary; mod dynamic_column; mod iterable; +mod payload_encoding; pub(crate) mod utils; mod value; @@ -46,6 +47,7 @@ pub use columnar::{ MergeRowOrder, ShuffleMergeOrder, StackMergeOrder, Version, compute_merged_term_ord_mapping, merge_columnar, }; +pub use payload_encoding::PayloadEncoding; use sstable::VoidSSTable; pub use value::{NumericalType, NumericalValue}; diff --git a/columnar/src/payload_encoding.rs b/columnar/src/payload_encoding.rs new file mode 100644 index 000000000..c7569b0bf --- /dev/null +++ b/columnar/src/payload_encoding.rs @@ -0,0 +1,12 @@ +use serde::{Deserialize, Serialize}; + +/// Encoding used to store string and byte column payloads. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum PayloadEncoding { + /// Store values in a dictionary and address them by ordinal. + #[default] + Dictionary, + /// Store values directly without assigning them dictionary ordinals. + Plain, +} diff --git a/src/schema/bytes_options.rs b/src/schema/bytes_options.rs index e5c28ea15..436fe42a1 100644 --- a/src/schema/bytes_options.rs +++ b/src/schema/bytes_options.rs @@ -3,13 +3,16 @@ use std::ops::BitOr; use serde::{Deserialize, Serialize}; use super::flags::{FastFlag, IndexedFlag, SchemaFlagList, StoredFlag}; +use crate::schema::PayloadEncoding; + /// Define how a bytes field should be handled by tantivy. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(from = "BytesOptionsDeser")] pub struct BytesOptions { indexed: bool, fieldnorms: bool, - fast: bool, + #[serde(serialize_with = "bytes_fast_field_options_serde::serialize")] + fast: Option, stored: bool, } @@ -23,7 +26,11 @@ struct BytesOptionsDeser { indexed: bool, #[serde(default)] fieldnorms: Option, - fast: bool, + #[serde( + default, + deserialize_with = "bytes_fast_field_options_serde::deserialize" + )] + fast: Option, stored: bool, } @@ -38,6 +45,45 @@ impl From for BytesOptions { } } +mod bytes_fast_field_options_serde { + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + use crate::schema::PayloadEncoding; + + #[derive(Serialize, Deserialize)] + #[serde(untagged)] + enum WireFormat { + IsEnabled(bool), + EnabledWithEncoding { encoding: PayloadEncoding }, + } + + pub fn serialize( + fast_field_encoding: &Option, + serializer: S, + ) -> Result + where + S: Serializer, + { + let wire_format = match fast_field_encoding { + None => WireFormat::IsEnabled(false), + Some(PayloadEncoding::Dictionary) => WireFormat::IsEnabled(true), + Some(encoding) => WireFormat::EnabledWithEncoding { + encoding: *encoding, + }, + }; + wire_format.serialize(serializer) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where D: Deserializer<'de> { + match WireFormat::deserialize(deserializer)? { + WireFormat::IsEnabled(false) => Ok(None), + WireFormat::IsEnabled(true) => Ok(Some(PayloadEncoding::Dictionary)), + WireFormat::EnabledWithEncoding { encoding } => Ok(Some(encoding)), + } + } +} + impl BytesOptions { /// Returns true if the value is indexed. #[inline] @@ -54,6 +100,12 @@ impl BytesOptions { /// Returns true if the value is a fast field. #[inline] pub fn is_fast(&self) -> bool { + self.fast.is_some() + } + + /// Returns the payload encoding if this is a fast field. + #[inline] + pub fn get_fast_field_encoding(&self) -> Option { self.fast } @@ -88,7 +140,14 @@ impl BytesOptions { /// Fast fields are designed for random access. #[must_use] pub fn set_fast(mut self) -> BytesOptions { - self.fast = true; + self.fast = Some(PayloadEncoding::Dictionary); + self + } + + /// Sets the field as a fast field using the supplied payload encoding. + #[must_use] + pub fn set_fast_with_encoding(mut self, encoding: PayloadEncoding) -> BytesOptions { + self.fast = Some(encoding); self } @@ -112,7 +171,15 @@ impl> BitOr for BytesOptions { indexed: self.indexed | other.indexed, fieldnorms: self.fieldnorms | other.fieldnorms, stored: self.stored | other.stored, - fast: self.fast | other.fast, + fast: match (self.fast, other.fast) { + (Some(PayloadEncoding::Plain), _) | (_, Some(PayloadEncoding::Plain)) => { + Some(PayloadEncoding::Plain) + } + (Some(PayloadEncoding::Dictionary), _) | (_, Some(PayloadEncoding::Dictionary)) => { + Some(PayloadEncoding::Dictionary) + } + (None, None) => None, + }, } } } @@ -129,7 +196,7 @@ impl From for BytesOptions { indexed: false, fieldnorms: false, stored: false, - fast: true, + fast: Some(PayloadEncoding::Dictionary), } } } @@ -140,7 +207,7 @@ impl From for BytesOptions { indexed: false, fieldnorms: false, stored: true, - fast: false, + fast: None, } } } @@ -151,7 +218,7 @@ impl From for BytesOptions { indexed: true, fieldnorms: true, stored: false, - fast: false, + fast: None, } } } @@ -169,7 +236,7 @@ where #[cfg(test)] mod tests { - use crate::schema::{BytesOptions, FAST, INDEXED, STORED}; + use crate::schema::{BytesOptions, PayloadEncoding, FAST, INDEXED, STORED}; #[test] fn test_bytes_option_fast_flag() { @@ -227,7 +294,7 @@ mod tests { &BytesOptions { indexed: true, fieldnorms: true, - fast: false, + fast: None, stored: false } ); @@ -246,7 +313,7 @@ mod tests { &BytesOptions { indexed: false, fieldnorms: false, - fast: false, + fast: None, stored: false } ); @@ -266,7 +333,7 @@ mod tests { &BytesOptions { indexed: true, fieldnorms: false, - fast: false, + fast: None, stored: false } ); @@ -287,9 +354,42 @@ mod tests { &BytesOptions { indexed: false, fieldnorms: true, - fast: false, + fast: None, stored: false } ); } + + #[test] + fn test_bytes_fast_encoding_serde() { + let dictionary = BytesOptions::default().set_fast(); + let dictionary_json = serde_json::to_value(&dictionary).unwrap(); + assert_eq!(dictionary_json["fast"], serde_json::json!(true)); + assert_eq!( + serde_json::from_value::(dictionary_json).unwrap(), + dictionary + ); + + let plain = BytesOptions::default().set_fast_with_encoding(PayloadEncoding::Plain); + let plain_json = serde_json::to_value(&plain).unwrap(); + assert_eq!( + plain_json["fast"], + serde_json::json!({ "encoding": "plain" }) + ); + assert_eq!( + serde_json::from_value::(plain_json).unwrap(), + plain + ); + } + + #[test] + fn test_fast_flag_does_not_override_plain_encoding() { + let plain = BytesOptions::default().set_fast_with_encoding(PayloadEncoding::Plain); + for options in [plain.clone() | FAST, BytesOptions::from(FAST) | plain] { + assert_eq!( + options.get_fast_field_encoding(), + Some(PayloadEncoding::Plain) + ); + } + } } diff --git a/src/schema/json_object_options.rs b/src/schema/json_object_options.rs index e66c7700b..c8a60d574 100644 --- a/src/schema/json_object_options.rs +++ b/src/schema/json_object_options.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use super::text_options::{merge_fast_field_options, FastFieldTextOptions}; use crate::schema::flags::{FastFlag, SchemaFlagList, StoredFlag}; -use crate::schema::{TextFieldIndexing, TextOptions}; +use crate::schema::{PayloadEncoding, TextFieldIndexing, TextOptions}; /// The `JsonObjectOptions` make it possible to /// configure how a json object field should be indexed and stored. @@ -75,6 +75,12 @@ impl JsonObjectOptions { .map(|fast_field_options| fast_field_options.tokenizer.as_str()) } + /// Returns the text fast-field options used by string subcolumns. + #[inline] + pub fn get_fast_field_options(&self) -> Option<&FastFieldTextOptions> { + self.fast.as_ref() + } + /// Returns `true` iff dots in json keys should be expanded. /// /// When expand_dots is enabled, json object like @@ -133,7 +139,17 @@ impl JsonObjectOptions { #[must_use] pub fn set_fast(mut self, tokenizer_name: impl ToString) -> Self { let tokenizer = tokenizer_name.to_string(); - self.fast = Some(FastFieldTextOptions { tokenizer }); + self.fast = Some(FastFieldTextOptions { + tokenizer, + encoding: PayloadEncoding::Dictionary, + }); + self + } + + /// Sets the field as a fast field using the supplied string-subcolumn options. + #[must_use] + pub fn set_fast_with_options(mut self, options: FastFieldTextOptions) -> Self { + self.fast = Some(options); self } @@ -265,4 +281,35 @@ mod tests { serde_json::json!(true) ); } + + #[test] + fn test_plain_fast_serde() { + let options = JsonObjectOptions::default().set_fast_with_options( + FastFieldTextOptions::new(RAW_TOKENIZER_NAME, PayloadEncoding::Plain), + ); + let serialized = serde_json::to_value(&options).unwrap(); + assert_eq!( + serialized["fast"], + serde_json::json!({ + "with_tokenizer": "raw", + "encoding": "plain" + }) + ); + assert_eq!( + serde_json::from_value::(serialized).unwrap(), + options + ); + } + + #[test] + fn test_fast_flag_does_not_override_plain_encoding() { + let plain = JsonObjectOptions::default() + .set_fast_with_options(FastFieldTextOptions::new("default", PayloadEncoding::Plain)); + for options in [plain.clone() | FAST, JsonObjectOptions::from(FAST) | plain] { + assert_eq!( + options.get_fast_field_options().unwrap(), + &FastFieldTextOptions::new("default", PayloadEncoding::Plain) + ); + } + } } diff --git a/src/schema/mod.rs b/src/schema/mod.rs index bfffba365..c386d1724 100644 --- a/src/schema/mod.rs +++ b/src/schema/mod.rs @@ -132,6 +132,7 @@ mod numeric_options; mod text_options; use columnar::ColumnType; +pub use columnar::PayloadEncoding; pub use self::bytes_options::BytesOptions; pub use self::custom_options::CustomOptions; @@ -151,7 +152,7 @@ pub use self::named_field_document::NamedFieldDocument; pub use self::numeric_options::NumericOptions; pub use self::schema::{Schema, SchemaBuilder}; pub use self::term::{Term, ValueBytes}; -pub use self::text_options::{TextFieldIndexing, TextOptions, STRING, TEXT}; +pub use self::text_options::{FastFieldTextOptions, TextFieldIndexing, TextOptions, STRING, TEXT}; /// Validator for a potential `field_name`. /// Returns true if the name can be use for a field name. diff --git a/src/schema/text_options.rs b/src/schema/text_options.rs index ee796b905..3bcf11aa3 100644 --- a/src/schema/text_options.rs +++ b/src/schema/text_options.rs @@ -1,6 +1,7 @@ use std::borrow::Cow; use std::ops::BitOr; +use columnar::PayloadEncoding; use serde::{Deserialize, Serialize}; use super::flags::{CoerceFlag, FastFlag}; @@ -26,27 +27,72 @@ pub struct TextOptions { } /// Options controlling how a text fast field is tokenized. -#[derive(Clone, Debug, PartialEq)] -pub(crate) struct FastFieldTextOptions { +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct FastFieldTextOptions { + /// Tokenizer applied before values are stored in the fast field. pub tokenizer: String, + /// Encoding used to store the resulting values. + #[serde(default)] + pub encoding: PayloadEncoding, } impl Default for FastFieldTextOptions { fn default() -> Self { FastFieldTextOptions { tokenizer: RAW_TOKENIZER_NAME.to_string(), + encoding: PayloadEncoding::Dictionary, } } } +impl FastFieldTextOptions { + /// Creates text fast-field options with the given tokenizer and payload encoding. + pub fn new(tokenizer: impl ToString, encoding: PayloadEncoding) -> Self { + FastFieldTextOptions { + tokenizer: tokenizer.to_string(), + encoding, + } + } + + /// Sets the tokenizer used by the text fast field. + #[must_use] + pub fn set_tokenizer(mut self, tokenizer: impl ToString) -> Self { + self.tokenizer = tokenizer.to_string(); + self + } + + /// Sets the payload encoding used by the text fast field. + #[must_use] + pub fn set_encoding(mut self, encoding: PayloadEncoding) -> Self { + self.encoding = encoding; + self + } +} + pub(super) fn merge_fast_field_options( left: Option, right: Option, ) -> Option { - // A configured tokenizer takes precedence over the implicit raw tokenizer. match (left, right) { - (Some(left), Some(right)) if left.tokenizer == RAW_TOKENIZER_NAME => Some(right), - (Some(left), _) => Some(left), + (Some(left), Some(right)) => { + // Merge tokenizer and encoding independently. This ensures that an implicit + // dictionary setting from FAST cannot overwrite an explicit plain encoding. + let tokenizer = if left.tokenizer == RAW_TOKENIZER_NAME { + right.tokenizer + } else { + left.tokenizer + }; + let encoding = if left.encoding == PayloadEncoding::Dictionary { + right.encoding + } else { + left.encoding + }; + Some(FastFieldTextOptions { + tokenizer, + encoding, + }) + } + (Some(left), None) => Some(left), (None, right) => right, } } @@ -54,13 +100,21 @@ pub(super) fn merge_fast_field_options( pub(super) mod fast_field_text_options_serde { use serde::{Deserialize, Deserializer, Serialize, Serializer}; - use super::{FastFieldTextOptions, RAW_TOKENIZER_NAME}; + use super::{FastFieldTextOptions, PayloadEncoding, RAW_TOKENIZER_NAME}; + + fn is_dictionary(encoding: &PayloadEncoding) -> bool { + *encoding == PayloadEncoding::Dictionary + } #[derive(Serialize, Deserialize)] #[serde(untagged)] enum WireFormat { IsEnabled(bool), - EnabledWithTokenizer { with_tokenizer: String }, + EnabledWithTokenizer { + with_tokenizer: String, + #[serde(default, skip_serializing_if = "is_dictionary")] + encoding: PayloadEncoding, + }, } pub fn serialize( @@ -72,11 +126,15 @@ pub(super) mod fast_field_text_options_serde { { let wire_format = match fast_field_options { None => WireFormat::IsEnabled(false), - Some(fast_field_options) if fast_field_options.tokenizer == RAW_TOKENIZER_NAME => { + Some(fast_field_options) + if fast_field_options.tokenizer == RAW_TOKENIZER_NAME + && fast_field_options.encoding == PayloadEncoding::Dictionary => + { WireFormat::IsEnabled(true) } Some(fast_field_options) => WireFormat::EnabledWithTokenizer { with_tokenizer: fast_field_options.tokenizer.clone(), + encoding: fast_field_options.encoding, }, }; wire_format.serialize(serializer) @@ -88,8 +146,12 @@ pub(super) mod fast_field_text_options_serde { match wire_format { WireFormat::IsEnabled(false) => Ok(None), WireFormat::IsEnabled(true) => Ok(Some(FastFieldTextOptions::default())), - WireFormat::EnabledWithTokenizer { with_tokenizer } => Ok(Some(FastFieldTextOptions { + WireFormat::EnabledWithTokenizer { + with_tokenizer, + encoding, + } => Ok(Some(FastFieldTextOptions { tokenizer: with_tokenizer, + encoding, })), } } @@ -127,6 +189,12 @@ impl TextOptions { .map(|fast_field_options| fast_field_options.tokenizer.as_str()) } + /// Returns the text fast-field options, if this is a fast field. + #[inline] + pub fn get_fast_field_options(&self) -> Option<&FastFieldTextOptions> { + self.fast.as_ref() + } + /// Returns true if values should be coerced to strings (numbers, null). #[inline] pub fn should_coerce(&self) -> bool { @@ -152,7 +220,17 @@ impl TextOptions { #[must_use] pub fn set_fast(mut self, tokenizer_name: impl ToString) -> TextOptions { let tokenizer = tokenizer_name.to_string(); - self.fast = Some(FastFieldTextOptions { tokenizer }); + self.fast = Some(FastFieldTextOptions { + tokenizer, + encoding: PayloadEncoding::Dictionary, + }); + self + } + + /// Sets the field as a fast field using the supplied tokenizer and payload encoding. + #[must_use] + pub fn set_fast_with_options(mut self, options: FastFieldTextOptions) -> TextOptions { + self.fast = Some(options); self } @@ -349,6 +427,7 @@ where mod tests { use crate::schema::text_options::FastFieldTextOptions; use crate::schema::*; + use crate::tokenizer::RAW_TOKENIZER_NAME; #[test] fn test_field_options() { @@ -393,6 +472,27 @@ mod tests { ); } + #[test] + fn test_fast_field_options_composition_preserves_plain_encoding() { + let plain_options = TextOptions::default().set_fast_with_options( + FastFieldTextOptions::default().set_encoding(PayloadEncoding::Plain), + ); + let tokenized_options = TextOptions::default().set_fast("default"); + + for options in [ + plain_options.clone() | tokenized_options.clone(), + tokenized_options | plain_options, + ] { + assert_eq!( + options.get_fast_field_options(), + Some(&FastFieldTextOptions::new( + "default", + PayloadEncoding::Plain + )) + ); + } + } + #[test] fn serde_default_test() { let json = r#" @@ -423,7 +523,8 @@ mod tests { assert_eq!( options.fast, Some(FastFieldTextOptions { - tokenizer: "default".to_string() + tokenizer: "default".to_string(), + encoding: PayloadEncoding::Dictionary, }) ); let serialized = serde_json::to_value(&options).unwrap(); @@ -435,7 +536,8 @@ mod tests { assert_eq!( options.fast, Some(FastFieldTextOptions { - tokenizer: "default".to_string() + tokenizer: "default".to_string(), + encoding: PayloadEncoding::Dictionary, }) ); @@ -446,7 +548,8 @@ mod tests { assert_eq!( options.fast, Some(FastFieldTextOptions { - tokenizer: "raw".to_string() + tokenizer: "raw".to_string(), + encoding: PayloadEncoding::Dictionary, }) ); let serialized = serde_json::to_value(&options).unwrap(); @@ -455,7 +558,8 @@ mod tests { assert_eq!( options.fast, Some(FastFieldTextOptions { - tokenizer: "raw".to_string() + tokenizer: "raw".to_string(), + encoding: PayloadEncoding::Dictionary, }) ); @@ -469,4 +573,40 @@ mod tests { let options: TextOptions = serde_json::from_value(serialized).unwrap(); assert_eq!(options.fast, None); } + + #[test] + fn serde_plain_fast_field_options() { + let options = TextOptions::default().set_fast_with_options(FastFieldTextOptions::new( + RAW_TOKENIZER_NAME, + PayloadEncoding::Plain, + )); + let serialized = serde_json::to_value(&options).unwrap(); + assert_eq!( + serialized["fast"], + serde_json::json!({ + "with_tokenizer": "raw", + "encoding": "plain" + }) + ); + assert_eq!( + serde_json::from_value::(serialized).unwrap(), + options + ); + } + + #[test] + fn fast_field_text_options_default_to_dictionary_encoding() { + assert_eq!( + FastFieldTextOptions::default().encoding, + PayloadEncoding::Dictionary + ); + assert_eq!( + TextOptions::default() + .set_fast(RAW_TOKENIZER_NAME) + .get_fast_field_options() + .unwrap() + .encoding, + PayloadEncoding::Dictionary + ); + } }