Added configuration for Plain/Dictionary encoding in fast bytes/text fields.

columnar/Cargo.toml

@@ -16,7 +16,7 @@ stacker = { version= "0.7", path = "../stacker", package="tantivy-stacker"}
 sstable = { version= "0.7", path = "../sstable", package = "tantivy-sstable" }
 common = { version= "0.11", path = "../common", package = "tantivy-common" }
 tantivy-bitpacker = { version= "0.10", path = "../bitpacker/" }
-serde = "1.0.152"
+serde = { version = "1.0.152", features = ["derive"] }
 downcast-rs = "2.0.1"
 
 [dev-dependencies]

columnar/src/lib.rs

@@ -32,6 +32,7 @@ mod columnar;
 mod dictionary;
 mod dynamic_column;
 mod iterable;
+mod payload_encoding;
 pub(crate) mod utils;
 mod value;
 
@@ -46,6 +47,7 @@ pub use columnar::{
     MergeRowOrder, ShuffleMergeOrder, StackMergeOrder, Version, compute_merged_term_ord_mapping,
     merge_columnar,
 };
+pub use payload_encoding::PayloadEncoding;
 use sstable::VoidSSTable;
 pub use value::{NumericalType, NumericalValue};
 

columnar/src/payload_encoding.rs

@@ -0,0 +1,12 @@
+use serde::{Deserialize, Serialize};
+
+/// Encoding used to store string and byte column payloads.
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum PayloadEncoding {
+    /// Store values in a dictionary and address them by ordinal.
+    #[default]
+    Dictionary,
+    /// Store values directly without assigning them dictionary ordinals.
+    Plain,
+}

src/schema/bytes_options.rs

@@ -3,13 +3,16 @@ use std::ops::BitOr;
 use serde::{Deserialize, Serialize};
 
 use super::flags::{FastFlag, IndexedFlag, SchemaFlagList, StoredFlag};
+use crate::schema::PayloadEncoding;
+
 /// Define how a bytes field should be handled by tantivy.
 #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(from = "BytesOptionsDeser")]
 pub struct BytesOptions {
     indexed: bool,
     fieldnorms: bool,
-    fast: bool,
+    #[serde(serialize_with = "bytes_fast_field_options_serde::serialize")]
+    fast: Option<PayloadEncoding>,
     stored: bool,
 }
 
@@ -23,7 +26,11 @@ struct BytesOptionsDeser {
     indexed: bool,
     #[serde(default)]
     fieldnorms: Option<bool>,
-    fast: bool,
+    #[serde(
+        default,
+        deserialize_with = "bytes_fast_field_options_serde::deserialize"
+    )]
+    fast: Option<PayloadEncoding>,
     stored: bool,
 }
 
@@ -38,6 +45,45 @@ impl From<BytesOptionsDeser> for BytesOptions {
     }
 }
 
+mod bytes_fast_field_options_serde {
+    use serde::{Deserialize, Deserializer, Serialize, Serializer};
+
+    use crate::schema::PayloadEncoding;
+
+    #[derive(Serialize, Deserialize)]
+    #[serde(untagged)]
+    enum WireFormat {
+        IsEnabled(bool),
+        EnabledWithEncoding { encoding: PayloadEncoding },
+    }
+
+    pub fn serialize<S>(
+        fast_field_encoding: &Option<PayloadEncoding>,
+        serializer: S,
+    ) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let wire_format = match fast_field_encoding {
+            None => WireFormat::IsEnabled(false),
+            Some(PayloadEncoding::Dictionary) => WireFormat::IsEnabled(true),
+            Some(encoding) => WireFormat::EnabledWithEncoding {
+                encoding: *encoding,
+            },
+        };
+        wire_format.serialize(serializer)
+    }
+
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<PayloadEncoding>, D::Error>
+    where D: Deserializer<'de> {
+        match WireFormat::deserialize(deserializer)? {
+            WireFormat::IsEnabled(false) => Ok(None),
+            WireFormat::IsEnabled(true) => Ok(Some(PayloadEncoding::Dictionary)),
+            WireFormat::EnabledWithEncoding { encoding } => Ok(Some(encoding)),
+        }
+    }
+}
+
 impl BytesOptions {
     /// Returns true if the value is indexed.
     #[inline]
@@ -54,6 +100,12 @@ impl BytesOptions {
     /// Returns true if the value is a fast field.
     #[inline]
     pub fn is_fast(&self) -> bool {
+        self.fast.is_some()
+    }
+
+    /// Returns the payload encoding if this is a fast field.
+    #[inline]
+    pub fn get_fast_field_encoding(&self) -> Option<PayloadEncoding> {
         self.fast
     }
 
@@ -88,7 +140,14 @@ impl BytesOptions {
     /// Fast fields are designed for random access.
     #[must_use]
     pub fn set_fast(mut self) -> BytesOptions {
-        self.fast = true;
+        self.fast = Some(PayloadEncoding::Dictionary);
+        self
+    }
+
+    /// Sets the field as a fast field using the supplied payload encoding.
+    #[must_use]
+    pub fn set_fast_with_encoding(mut self, encoding: PayloadEncoding) -> BytesOptions {
+        self.fast = Some(encoding);
         self
     }
 
@@ -112,7 +171,15 @@ impl<T: Into<BytesOptions>> BitOr<T> for BytesOptions {
             indexed: self.indexed | other.indexed,
             fieldnorms: self.fieldnorms | other.fieldnorms,
             stored: self.stored | other.stored,
-            fast: self.fast | other.fast,
+            fast: match (self.fast, other.fast) {
+                (Some(PayloadEncoding::Plain), _) | (_, Some(PayloadEncoding::Plain)) => {
+                    Some(PayloadEncoding::Plain)
+                }
+                (Some(PayloadEncoding::Dictionary), _) | (_, Some(PayloadEncoding::Dictionary)) => {
+                    Some(PayloadEncoding::Dictionary)
+                }
+                (None, None) => None,
+            },
         }
     }
 }
@@ -129,7 +196,7 @@ impl From<FastFlag> for BytesOptions {
             indexed: false,
             fieldnorms: false,
             stored: false,
-            fast: true,
+            fast: Some(PayloadEncoding::Dictionary),
         }
     }
 }
@@ -140,7 +207,7 @@ impl From<StoredFlag> for BytesOptions {
             indexed: false,
             fieldnorms: false,
             stored: true,
-            fast: false,
+            fast: None,
         }
     }
 }
@@ -151,7 +218,7 @@ impl From<IndexedFlag> for BytesOptions {
             indexed: true,
             fieldnorms: true,
             stored: false,
-            fast: false,
+            fast: None,
         }
     }
 }
@@ -169,7 +236,7 @@ where
 
 #[cfg(test)]
 mod tests {
-    use crate::schema::{BytesOptions, FAST, INDEXED, STORED};
+    use crate::schema::{BytesOptions, PayloadEncoding, FAST, INDEXED, STORED};
 
     #[test]
     fn test_bytes_option_fast_flag() {
@@ -227,7 +294,7 @@ mod tests {
             &BytesOptions {
                 indexed: true,
                 fieldnorms: true,
-                fast: false,
+                fast: None,
                 stored: false
             }
         );
@@ -246,7 +313,7 @@ mod tests {
             &BytesOptions {
                 indexed: false,
                 fieldnorms: false,
-                fast: false,
+                fast: None,
                 stored: false
             }
         );
@@ -266,7 +333,7 @@ mod tests {
             &BytesOptions {
                 indexed: true,
                 fieldnorms: false,
-                fast: false,
+                fast: None,
                 stored: false
             }
         );
@@ -287,9 +354,42 @@ mod tests {
             &BytesOptions {
                 indexed: false,
                 fieldnorms: true,
-                fast: false,
+                fast: None,
                 stored: false
             }
         );
     }
+
+    #[test]
+    fn test_bytes_fast_encoding_serde() {
+        let dictionary = BytesOptions::default().set_fast();
+        let dictionary_json = serde_json::to_value(&dictionary).unwrap();
+        assert_eq!(dictionary_json["fast"], serde_json::json!(true));
+        assert_eq!(
+            serde_json::from_value::<BytesOptions>(dictionary_json).unwrap(),
+            dictionary
+        );
+
+        let plain = BytesOptions::default().set_fast_with_encoding(PayloadEncoding::Plain);
+        let plain_json = serde_json::to_value(&plain).unwrap();
+        assert_eq!(
+            plain_json["fast"],
+            serde_json::json!({ "encoding": "plain" })
+        );
+        assert_eq!(
+            serde_json::from_value::<BytesOptions>(plain_json).unwrap(),
+            plain
+        );
+    }
+
+    #[test]
+    fn test_fast_flag_does_not_override_plain_encoding() {
+        let plain = BytesOptions::default().set_fast_with_encoding(PayloadEncoding::Plain);
+        for options in [plain.clone() | FAST, BytesOptions::from(FAST) | plain] {
+            assert_eq!(
+                options.get_fast_field_encoding(),
+                Some(PayloadEncoding::Plain)
+            );
+        }
+    }
 }

src/schema/json_object_options.rs

@@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize};
 
 use super::text_options::{merge_fast_field_options, FastFieldTextOptions};
 use crate::schema::flags::{FastFlag, SchemaFlagList, StoredFlag};
-use crate::schema::{TextFieldIndexing, TextOptions};
+use crate::schema::{PayloadEncoding, TextFieldIndexing, TextOptions};
 
 /// The `JsonObjectOptions` make it possible to
 /// configure how a json object field should be indexed and stored.
@@ -75,6 +75,12 @@ impl JsonObjectOptions {
             .map(|fast_field_options| fast_field_options.tokenizer.as_str())
     }
 
+    /// Returns the text fast-field options used by string subcolumns.
+    #[inline]
+    pub fn get_fast_field_options(&self) -> Option<&FastFieldTextOptions> {
+        self.fast.as_ref()
+    }
+
     /// Returns `true` iff dots in json keys should be expanded.
     ///
     /// When expand_dots is enabled, json object like
@@ -133,7 +139,17 @@ impl JsonObjectOptions {
     #[must_use]
     pub fn set_fast(mut self, tokenizer_name: impl ToString) -> Self {
         let tokenizer = tokenizer_name.to_string();
-        self.fast = Some(FastFieldTextOptions { tokenizer });
+        self.fast = Some(FastFieldTextOptions {
+            tokenizer,
+            encoding: PayloadEncoding::Dictionary,
+        });
+        self
+    }
+
+    /// Sets the field as a fast field using the supplied string-subcolumn options.
+    #[must_use]
+    pub fn set_fast_with_options(mut self, options: FastFieldTextOptions) -> Self {
+        self.fast = Some(options);
         self
     }
 
@@ -265,4 +281,35 @@ mod tests {
             serde_json::json!(true)
         );
     }
+
+    #[test]
+    fn test_plain_fast_serde() {
+        let options = JsonObjectOptions::default().set_fast_with_options(
+            FastFieldTextOptions::new(RAW_TOKENIZER_NAME, PayloadEncoding::Plain),
+        );
+        let serialized = serde_json::to_value(&options).unwrap();
+        assert_eq!(
+            serialized["fast"],
+            serde_json::json!({
+                "with_tokenizer": "raw",
+                "encoding": "plain"
+            })
+        );
+        assert_eq!(
+            serde_json::from_value::<JsonObjectOptions>(serialized).unwrap(),
+            options
+        );
+    }
+
+    #[test]
+    fn test_fast_flag_does_not_override_plain_encoding() {
+        let plain = JsonObjectOptions::default()
+            .set_fast_with_options(FastFieldTextOptions::new("default", PayloadEncoding::Plain));
+        for options in [plain.clone() | FAST, JsonObjectOptions::from(FAST) | plain] {
+            assert_eq!(
+                options.get_fast_field_options().unwrap(),
+                &FastFieldTextOptions::new("default", PayloadEncoding::Plain)
+            );
+        }
+    }
 }

src/schema/mod.rs

@@ -132,6 +132,7 @@ mod numeric_options;
 mod text_options;
 
 use columnar::ColumnType;
+pub use columnar::PayloadEncoding;
 
 pub use self::bytes_options::BytesOptions;
 pub use self::custom_options::CustomOptions;
@@ -151,7 +152,7 @@ pub use self::named_field_document::NamedFieldDocument;
 pub use self::numeric_options::NumericOptions;
 pub use self::schema::{Schema, SchemaBuilder};
 pub use self::term::{Term, ValueBytes};
-pub use self::text_options::{TextFieldIndexing, TextOptions, STRING, TEXT};
+pub use self::text_options::{FastFieldTextOptions, TextFieldIndexing, TextOptions, STRING, TEXT};
 
 /// Validator for a potential `field_name`.
 /// Returns true if the name can be use for a field name.

src/schema/text_options.rs

@@ -1,6 +1,7 @@
 use std::borrow::Cow;
 use std::ops::BitOr;
 
+use columnar::PayloadEncoding;
 use serde::{Deserialize, Serialize};
 
 use super::flags::{CoerceFlag, FastFlag};
@@ -26,27 +27,72 @@ pub struct TextOptions {
 }
 
 /// Options controlling how a text fast field is tokenized.
-#[derive(Clone, Debug, PartialEq)]
-pub(crate) struct FastFieldTextOptions {
+#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+pub struct FastFieldTextOptions {
+    /// Tokenizer applied before values are stored in the fast field.
     pub tokenizer: String,
+    /// Encoding used to store the resulting values.
+    #[serde(default)]
+    pub encoding: PayloadEncoding,
 }
 
 impl Default for FastFieldTextOptions {
     fn default() -> Self {
         FastFieldTextOptions {
             tokenizer: RAW_TOKENIZER_NAME.to_string(),
+            encoding: PayloadEncoding::Dictionary,
         }
     }
 }
 
+impl FastFieldTextOptions {
+    /// Creates text fast-field options with the given tokenizer and payload encoding.
+    pub fn new(tokenizer: impl ToString, encoding: PayloadEncoding) -> Self {
+        FastFieldTextOptions {
+            tokenizer: tokenizer.to_string(),
+            encoding,
+        }
+    }
+
+    /// Sets the tokenizer used by the text fast field.
+    #[must_use]
+    pub fn set_tokenizer(mut self, tokenizer: impl ToString) -> Self {
+        self.tokenizer = tokenizer.to_string();
+        self
+    }
+
+    /// Sets the payload encoding used by the text fast field.
+    #[must_use]
+    pub fn set_encoding(mut self, encoding: PayloadEncoding) -> Self {
+        self.encoding = encoding;
+        self
+    }
+}
+
 pub(super) fn merge_fast_field_options(
     left: Option<FastFieldTextOptions>,
     right: Option<FastFieldTextOptions>,
 ) -> Option<FastFieldTextOptions> {
-    // A configured tokenizer takes precedence over the implicit raw tokenizer.
     match (left, right) {
-        (Some(left), Some(right)) if left.tokenizer == RAW_TOKENIZER_NAME => Some(right),
-        (Some(left), _) => Some(left),
+        (Some(left), Some(right)) => {
+            // Merge tokenizer and encoding independently. This ensures that an implicit
+            // dictionary setting from FAST cannot overwrite an explicit plain encoding.
+            let tokenizer = if left.tokenizer == RAW_TOKENIZER_NAME {
+                right.tokenizer
+            } else {
+                left.tokenizer
+            };
+            let encoding = if left.encoding == PayloadEncoding::Dictionary {
+                right.encoding
+            } else {
+                left.encoding
+            };
+            Some(FastFieldTextOptions {
+                tokenizer,
+                encoding,
+            })
+        }
+        (Some(left), None) => Some(left),
         (None, right) => right,
     }
 }
@@ -54,13 +100,21 @@ pub(super) fn merge_fast_field_options(
 pub(super) mod fast_field_text_options_serde {
     use serde::{Deserialize, Deserializer, Serialize, Serializer};
 
-    use super::{FastFieldTextOptions, RAW_TOKENIZER_NAME};
+    use super::{FastFieldTextOptions, PayloadEncoding, RAW_TOKENIZER_NAME};
+
+    fn is_dictionary(encoding: &PayloadEncoding) -> bool {
+        *encoding == PayloadEncoding::Dictionary
+    }
 
     #[derive(Serialize, Deserialize)]
     #[serde(untagged)]
     enum WireFormat {
         IsEnabled(bool),
-        EnabledWithTokenizer { with_tokenizer: String },
+        EnabledWithTokenizer {
+            with_tokenizer: String,
+            #[serde(default, skip_serializing_if = "is_dictionary")]
+            encoding: PayloadEncoding,
+        },
     }
 
     pub fn serialize<S>(
@@ -72,11 +126,15 @@ pub(super) mod fast_field_text_options_serde {
     {
         let wire_format = match fast_field_options {
             None => WireFormat::IsEnabled(false),
-            Some(fast_field_options) if fast_field_options.tokenizer == RAW_TOKENIZER_NAME => {
+            Some(fast_field_options)
+                if fast_field_options.tokenizer == RAW_TOKENIZER_NAME
+                    && fast_field_options.encoding == PayloadEncoding::Dictionary =>
+            {
                 WireFormat::IsEnabled(true)
             }
             Some(fast_field_options) => WireFormat::EnabledWithTokenizer {
                 with_tokenizer: fast_field_options.tokenizer.clone(),
+                encoding: fast_field_options.encoding,
             },
         };
         wire_format.serialize(serializer)
@@ -88,8 +146,12 @@ pub(super) mod fast_field_text_options_serde {
         match wire_format {
             WireFormat::IsEnabled(false) => Ok(None),
             WireFormat::IsEnabled(true) => Ok(Some(FastFieldTextOptions::default())),
-            WireFormat::EnabledWithTokenizer { with_tokenizer } => Ok(Some(FastFieldTextOptions {
+            WireFormat::EnabledWithTokenizer {
+                with_tokenizer,
+                encoding,
+            } => Ok(Some(FastFieldTextOptions {
                 tokenizer: with_tokenizer,
+                encoding,
             })),
         }
     }
@@ -127,6 +189,12 @@ impl TextOptions {
             .map(|fast_field_options| fast_field_options.tokenizer.as_str())
     }
 
+    /// Returns the text fast-field options, if this is a fast field.
+    #[inline]
+    pub fn get_fast_field_options(&self) -> Option<&FastFieldTextOptions> {
+        self.fast.as_ref()
+    }
+
     /// Returns true if values should be coerced to strings (numbers, null).
     #[inline]
     pub fn should_coerce(&self) -> bool {
@@ -152,7 +220,17 @@ impl TextOptions {
     #[must_use]
     pub fn set_fast(mut self, tokenizer_name: impl ToString) -> TextOptions {
         let tokenizer = tokenizer_name.to_string();
-        self.fast = Some(FastFieldTextOptions { tokenizer });
+        self.fast = Some(FastFieldTextOptions {
+            tokenizer,
+            encoding: PayloadEncoding::Dictionary,
+        });
+        self
+    }
+
+    /// Sets the field as a fast field using the supplied tokenizer and payload encoding.
+    #[must_use]
+    pub fn set_fast_with_options(mut self, options: FastFieldTextOptions) -> TextOptions {
+        self.fast = Some(options);
         self
     }
 
@@ -349,6 +427,7 @@ where
 mod tests {
     use crate::schema::text_options::FastFieldTextOptions;
     use crate::schema::*;
+    use crate::tokenizer::RAW_TOKENIZER_NAME;
 
     #[test]
     fn test_field_options() {
@@ -393,6 +472,27 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_fast_field_options_composition_preserves_plain_encoding() {
+        let plain_options = TextOptions::default().set_fast_with_options(
+            FastFieldTextOptions::default().set_encoding(PayloadEncoding::Plain),
+        );
+        let tokenized_options = TextOptions::default().set_fast("default");
+
+        for options in [
+            plain_options.clone() | tokenized_options.clone(),
+            tokenized_options | plain_options,
+        ] {
+            assert_eq!(
+                options.get_fast_field_options(),
+                Some(&FastFieldTextOptions::new(
+                    "default",
+                    PayloadEncoding::Plain
+                ))
+            );
+        }
+    }
+
     #[test]
     fn serde_default_test() {
         let json = r#"
@@ -423,7 +523,8 @@ mod tests {
         assert_eq!(
             options.fast,
             Some(FastFieldTextOptions {
-                tokenizer: "default".to_string()
+                tokenizer: "default".to_string(),
+                encoding: PayloadEncoding::Dictionary,
             })
         );
         let serialized = serde_json::to_value(&options).unwrap();
@@ -435,7 +536,8 @@ mod tests {
         assert_eq!(
             options.fast,
             Some(FastFieldTextOptions {
-                tokenizer: "default".to_string()
+                tokenizer: "default".to_string(),
+                encoding: PayloadEncoding::Dictionary,
             })
         );
 
@@ -446,7 +548,8 @@ mod tests {
         assert_eq!(
             options.fast,
             Some(FastFieldTextOptions {
-                tokenizer: "raw".to_string()
+                tokenizer: "raw".to_string(),
+                encoding: PayloadEncoding::Dictionary,
             })
         );
         let serialized = serde_json::to_value(&options).unwrap();
@@ -455,7 +558,8 @@ mod tests {
         assert_eq!(
             options.fast,
             Some(FastFieldTextOptions {
-                tokenizer: "raw".to_string()
+                tokenizer: "raw".to_string(),
+                encoding: PayloadEncoding::Dictionary,
             })
         );
 
@@ -469,4 +573,40 @@ mod tests {
         let options: TextOptions = serde_json::from_value(serialized).unwrap();
         assert_eq!(options.fast, None);
     }
+
+    #[test]
+    fn serde_plain_fast_field_options() {
+        let options = TextOptions::default().set_fast_with_options(FastFieldTextOptions::new(
+            RAW_TOKENIZER_NAME,
+            PayloadEncoding::Plain,
+        ));
+        let serialized = serde_json::to_value(&options).unwrap();
+        assert_eq!(
+            serialized["fast"],
+            serde_json::json!({
+                "with_tokenizer": "raw",
+                "encoding": "plain"
+            })
+        );
+        assert_eq!(
+            serde_json::from_value::<TextOptions>(serialized).unwrap(),
+            options
+        );
+    }
+
+    #[test]
+    fn fast_field_text_options_default_to_dictionary_encoding() {
+        assert_eq!(
+            FastFieldTextOptions::default().encoding,
+            PayloadEncoding::Dictionary
+        );
+        assert_eq!(
+            TextOptions::default()
+                .set_fast(RAW_TOKENIZER_NAME)
+                .get_fast_field_options()
+                .unwrap()
+                .encoding,
+            PayloadEncoding::Dictionary
+        );
+    }
 }