From 786781d0fc500c6445d87344d2abd8b66242bfbd Mon Sep 17 00:00:00 2001 From: Pascal Seitz Date: Sat, 20 Apr 2024 21:21:19 +0800 Subject: [PATCH] cleanup --- src/core/json_utils.rs | 24 +- src/indexer/segment_writer.rs | 9 +- src/postings/postings_writer.rs | 41 +-- src/query/fuzzy_query.rs | 45 ++- .../phrase_prefix_query.rs | 2 +- src/schema/indexing_term.rs | 123 ++++---- src/schema/term.rs | 272 +++--------------- 7 files changed, 158 insertions(+), 358 deletions(-) diff --git a/src/core/json_utils.rs b/src/core/json_utils.rs index ae04cf43f..f18fb7514 100644 --- a/src/core/json_utils.rs +++ b/src/core/json_utils.rs @@ -141,9 +141,6 @@ fn index_json_value<'a, V: Value<'a>>( term_buffer.truncate_value_bytes(0); term_buffer.append_bytes(&unordered_id.to_be_bytes()); }; - let set_type = |term_buffer: &mut IndexingTerm, typ: Type| { - term_buffer.append_bytes(&[typ.to_code()]); - }; match json_value.as_value() { ReferenceValue::Leaf(leaf) => match leaf { @@ -156,7 +153,7 @@ fn index_json_value<'a, V: Value<'a>>( // TODO: make sure the chain position works out. set_path_id(term_buffer, unordered_id); - set_type(term_buffer, Type::Str); + term_buffer.append_bytes(&[Type::Str.to_code()]); let indexing_position = positions_per_path.get_position_from_id(unordered_id); postings_writer.index_text( doc, @@ -262,8 +259,9 @@ pub(crate) fn convert_to_fast_value_and_append_to_json_term( ) -> Option { assert_eq!( term.value() - .as_json_value_bytes() + .as_json() .expect("expecting a Term with a json type and json path") + .1 .as_serialized() .len(), 0, @@ -409,8 +407,8 @@ mod tests { term.append_type_and_fast_value(-4i64); assert_eq!( - term.serialized_term(), - b"\x00\x00\x00\x01jcolor\x00i\x7f\xff\xff\xff\xff\xff\xff\xfc" + term.value().as_serialized(), + b"jcolor\x00i\x7f\xff\xff\xff\xff\xff\xff\xfc" ) } @@ -421,8 +419,8 @@ mod tests { term.append_type_and_fast_value(4u64); assert_eq!( - term.serialized_term(), - b"\x00\x00\x00\x01jcolor\x00u\x00\x00\x00\x00\x00\x00\x00\x04" + term.value().as_serialized(), + b"jcolor\x00u\x00\x00\x00\x00\x00\x00\x00\x04" ) } @@ -432,8 +430,8 @@ mod tests { let mut term = term_from_json_paths(field, ["color"].into_iter(), false); term.append_type_and_fast_value(4.0f64); assert_eq!( - term.serialized_term(), - b"\x00\x00\x00\x01jcolor\x00f\xc0\x10\x00\x00\x00\x00\x00\x00" + term.value().as_serialized(), + b"jcolor\x00f\xc0\x10\x00\x00\x00\x00\x00\x00" ) } @@ -443,8 +441,8 @@ mod tests { let mut term = term_from_json_paths(field, ["color"].into_iter(), false); term.append_type_and_fast_value(true); assert_eq!( - term.serialized_term(), - b"\x00\x00\x00\x01jcolor\x00o\x00\x00\x00\x00\x00\x00\x00\x01" + term.value().as_serialized(), + b"jcolor\x00o\x00\x00\x00\x00\x00\x00\x00\x01" ) } diff --git a/src/indexer/segment_writer.rs b/src/indexer/segment_writer.rs index 43cdf5fc5..0762b9732 100644 --- a/src/indexer/segment_writer.rs +++ b/src/indexer/segment_writer.rs @@ -127,7 +127,7 @@ impl SegmentWriter { )?, doc_opstamps: Vec::with_capacity(1_000), per_field_text_analyzers, - term_buffer: IndexingTerm::with_capacity(16), + term_buffer: IndexingTerm::new(), schema, }) } @@ -196,7 +196,7 @@ impl SegmentWriter { let (term_buffer, ctx) = (&mut self.term_buffer, &mut self.ctx); let postings_writer: &mut dyn PostingsWriter = self.per_field_postings_writers.get_for_field_mut(field); - term_buffer.clear_with_field_and_type(field_entry.field_type().value_type(), field); + term_buffer.clear_with_field(field); match field_entry.field_type() { FieldType::Facet(_) => { @@ -272,8 +272,7 @@ impl SegmentWriter { num_vals += 1; let date_val = value.as_datetime().ok_or_else(make_schema_error)?; - term_buffer - .set_u64(date_val.truncate(DATE_TIME_PRECISION_INDEXED).to_u64()); + term_buffer.set_date(date_val); postings_writer.subscribe(doc_id, 0u32, term_buffer, ctx); } if field_entry.has_fieldnorms() { @@ -333,7 +332,7 @@ impl SegmentWriter { num_vals += 1; let bytes = value.as_bytes().ok_or_else(make_schema_error)?; - term_buffer.set_bytes(bytes); + term_buffer.set_value_bytes(bytes); postings_writer.subscribe(doc_id, 0u32, term_buffer, ctx); } if field_entry.has_fieldnorms() { diff --git a/src/postings/postings_writer.rs b/src/postings/postings_writer.rs index f95524f65..5802f74f3 100644 --- a/src/postings/postings_writer.rs +++ b/src/postings/postings_writer.rs @@ -11,7 +11,7 @@ use crate::postings::recorder::{BufferLender, Recorder}; use crate::postings::{ FieldSerializer, IndexingContext, InvertedIndexSerializer, PerFieldPostingsWriter, }; -use crate::schema::indexing_term::IndexingTerm; +use crate::schema::indexing_term::{get_field_from_indexing_term, IndexingTerm}; use crate::schema::{Field, Schema, Type}; use crate::tokenizer::{Token, TokenStream, MAX_TOKEN_LEN}; use crate::DocId; @@ -61,14 +61,14 @@ pub(crate) fn serialize_postings( let mut term_offsets: Vec<(Field, OrderedPathId, &[u8], Addr)> = Vec::with_capacity(ctx.term_index.len()); term_offsets.extend(ctx.term_index.iter().map(|(key, addr)| { - let field = IndexingTerm::wrap(key).field(); + let field = get_field_from_indexing_term(key); if schema.get_field_entry(field).field_type().value_type() == Type::Json { - let byte_range_path = 5..5 + 4; + let byte_range_path = 4..4 + 4; let unordered_id = u32::from_be_bytes(key[byte_range_path.clone()].try_into().unwrap()); let path_id = unordered_id_to_ordered_id[unordered_id as usize]; (field, path_id, &key[byte_range_path.end..], addr) } else { - (field, 0.into(), &key[5..], addr) + (field, 0.into(), &key[4..], addr) } })); // Sort by field, path, and term @@ -211,25 +211,28 @@ impl PostingsWriter for SpecializedPostingsWriter { term: &IndexingTerm, ctx: &mut IndexingContext, ) { - debug_assert!(term.serialized_term().len() >= 4); + debug_assert!(term.serialized_for_hashmap().len() >= 4); self.total_num_tokens += 1; let (term_index, arena) = (&mut ctx.term_index, &mut ctx.arena); - term_index.mutate_or_create(term.serialized_term(), |opt_recorder: Option| { - if let Some(mut recorder) = opt_recorder { - let current_doc = recorder.current_doc(); - if current_doc != doc { - recorder.close_doc(arena); + term_index.mutate_or_create( + term.serialized_for_hashmap(), + |opt_recorder: Option| { + if let Some(mut recorder) = opt_recorder { + let current_doc = recorder.current_doc(); + if current_doc != doc { + recorder.close_doc(arena); + recorder.new_doc(doc, arena); + } + recorder.record_position(position, arena); + recorder + } else { + let mut recorder = Rec::default(); recorder.new_doc(doc, arena); + recorder.record_position(position, arena); + recorder } - recorder.record_position(position, arena); - recorder - } else { - let mut recorder = Rec::default(); - recorder.new_doc(doc, arena); - recorder.record_position(position, arena); - recorder - } - }); + }, + ); } fn serialize( diff --git a/src/query/fuzzy_query.rs b/src/query/fuzzy_query.rs index a2e3f2a6b..21b5aace0 100644 --- a/src/query/fuzzy_query.rs +++ b/src/query/fuzzy_query.rs @@ -3,7 +3,7 @@ use once_cell::sync::OnceCell; use tantivy_fst::Automaton; use crate::query::{AutomatonWeight, EnableScoring, Query, Weight}; -use crate::schema::{Term, Type}; +use crate::schema::Term; use crate::TantivyError::InvalidArgument; pub(crate) struct DfaWrapper(pub DFA); @@ -133,40 +133,33 @@ impl FuzzyTermQuery { let term_value = self.term.value(); - let term_text = if term_value.typ() == Type::Json { - if let Some(json_path_type) = term_value.json_path_type() { - if json_path_type != Type::Str { - return Err(InvalidArgument(format!( - "The fuzzy term query requires a string path type for a json term. Found \ - {:?}", - json_path_type - ))); - } + let get_automaton = |term_text: &str| { + if self.prefix { + automaton_builder.build_prefix_dfa(term_text) + } else { + automaton_builder.build_dfa(term_text) } - - std::str::from_utf8(self.term.serialized_value_bytes()).map_err(|_| { - InvalidArgument( - "Failed to convert json term value bytes to utf8 string.".to_string(), - ) - })? - } else { - term_value.as_str().ok_or_else(|| { - InvalidArgument("The fuzzy term query requires a string term.".to_string()) - })? - }; - let automaton = if self.prefix { - automaton_builder.build_prefix_dfa(term_text) - } else { - automaton_builder.build_dfa(term_text) }; - if let Some((json_path_bytes, _)) = term_value.as_json() { + if let Some((json_path_bytes, _term_value)) = term_value.as_json() { + let term_text = + std::str::from_utf8(self.term.serialized_value_bytes()).map_err(|_| { + InvalidArgument( + "Failed to convert json term value bytes to utf8 string.".to_string(), + ) + })?; + + let automaton = get_automaton(term_text); Ok(AutomatonWeight::new_for_json_path( self.term.field(), DfaWrapper(automaton), json_path_bytes, )) } else { + let term_text = term_value.as_str().ok_or_else(|| { + InvalidArgument("The fuzzy term query requires a string term.".to_string()) + })?; + let automaton = get_automaton(term_text); Ok(AutomatonWeight::new( self.term.field(), DfaWrapper(automaton), diff --git a/src/query/phrase_prefix_query/phrase_prefix_query.rs b/src/query/phrase_prefix_query/phrase_prefix_query.rs index 8cbbe637e..c428d2b30 100644 --- a/src/query/phrase_prefix_query/phrase_prefix_query.rs +++ b/src/query/phrase_prefix_query/phrase_prefix_query.rs @@ -137,7 +137,7 @@ impl Query for PhrasePrefixQuery { // There are no prefix. Let's just match the suffix. let end_term = if let Some(end_value) = prefix_end(self.prefix.1.serialized_value_bytes()) { - let mut end_term = Term::with_capacity(end_value.len()); + let mut end_term = Term::new(); end_term.set_field_and_type(self.field, self.prefix.1.typ()); end_term.append_bytes(&end_value); Bound::Excluded(end_term) diff --git a/src/schema/indexing_term.rs b/src/schema/indexing_term.rs index db2d0a153..ffe6494e3 100644 --- a/src/schema/indexing_term.rs +++ b/src/schema/indexing_term.rs @@ -8,48 +8,42 @@ use crate::fastfield::FastValue; use crate::schema::Type; use crate::DateTime; -/// Term represents the value that the token can take. +/// IndexingTerm represents is the serialized information of a term during indexing. /// It's a serialized representation over different types. /// -/// It actually wraps a `Vec`. The first 5 bytes are metadata. -/// 4 bytes are the field id, and the last byte is the type. +/// It actually wraps a `Vec`. /// -/// The serialized value `ValueBytes` is considered everything after the 4 first bytes (term id). +/// The format is as follow: +/// `[field id: u32][serialized value]` +/// +/// For JSON it equals to: +/// `[field id: u32][path id: u32][type code: u8][serialized value]` +/// +/// The format is chosen to easily partition the terms by field during serialization, as all terms +/// are stored in one hashmap. #[derive(Clone)] -pub struct IndexingTerm>(B) -where - B: AsRef<[u8]>; +pub(crate) struct IndexingTerm(Vec); -/// The number of bytes used as metadata by `Term`. -const TERM_METADATA_LENGTH: usize = 5; +/// The number of bytes used as for the field id by `Term`. +const FIELD_ID_LENGTH: usize = 4; impl IndexingTerm { - /// Create a new Term with a buffer with a given capacity. - pub fn with_capacity(capacity: usize) -> IndexingTerm { - let mut data = Vec::with_capacity(TERM_METADATA_LENGTH + capacity); - data.resize(TERM_METADATA_LENGTH, 0u8); + /// Create a new IndexingTerm. + pub fn new() -> IndexingTerm { + let mut data = Vec::with_capacity(FIELD_ID_LENGTH + 32); + data.resize(FIELD_ID_LENGTH, 0u8); IndexingTerm(data) } - /// Panics when the term is not empty... ie: some value is set. - /// Use `clear_with_field_and_type` in that case. - /// - /// Sets field and the type. - pub(crate) fn set_field_and_type(&mut self, field: Field, typ: Type) { - assert!(self.is_empty()); - self.0[0..4].clone_from_slice(field.field_id().to_be_bytes().as_ref()); - self.0[4] = typ.to_code(); - } - /// Is empty if there are no value bytes. pub fn is_empty(&self) -> bool { - self.0.len() == TERM_METADATA_LENGTH + self.0.len() == FIELD_ID_LENGTH } - /// Removes the value_bytes and set the field and type code. - pub(crate) fn clear_with_field_and_type(&mut self, typ: Type, field: Field) { + /// Removes the value_bytes and set the field + pub(crate) fn clear_with_field(&mut self, field: Field) { self.truncate_value_bytes(0); - self.set_field_and_type(field, typ); + self.0[0..4].clone_from_slice(field.field_id().to_be_bytes().as_ref()); } /// Sets a u64 value in the term. @@ -62,6 +56,11 @@ impl IndexingTerm { self.set_fast_value(val); } + /// Sets a `DateTime` value in the term. + pub fn set_date(&mut self, val: DateTime) { + self.set_fast_value(val); + } + /// Sets a `i64` value in the term. pub fn set_i64(&mut self, val: i64) { self.set_fast_value(val); @@ -78,7 +77,19 @@ impl IndexingTerm { } fn set_fast_value(&mut self, val: T) { - self.set_bytes(val.to_u64().to_be_bytes().as_ref()); + self.truncate_value_bytes(0); + self.append_fast_value(val); + } + + /// Sets a `Ipv6Addr` value in the term. + pub fn set_ip_addr(&mut self, val: Ipv6Addr) { + self.set_value_bytes(val.to_u128().to_be_bytes().as_ref()); + } + + /// Sets the value bytes of the term. + pub fn set_value_bytes(&mut self, bytes: &[u8]) { + self.truncate_value_bytes(0); + self.0.extend(bytes); } /// Append a type marker + fast value to a term. @@ -87,6 +98,13 @@ impl IndexingTerm { /// It will not clear existing bytes. pub(crate) fn append_type_and_fast_value(&mut self, val: T) { self.0.push(T::to_type().to_code()); + self.append_fast_value(val) + } + + /// Append a fast value to a term. + /// + /// It will not clear existing bytes. + pub fn append_fast_value(&mut self, val: T) { let value = if T::to_type() == Type::Date { DateTime::from_u64(val.to_u64()) .truncate(DATE_TIME_PRECISION_INDEXED) @@ -97,60 +115,33 @@ impl IndexingTerm { self.0.extend(value.to_be_bytes().as_ref()); } - /// Sets a `Ipv6Addr` value in the term. - pub fn set_ip_addr(&mut self, val: Ipv6Addr) { - self.set_bytes(val.to_u128().to_be_bytes().as_ref()); - } - - /// Sets the value of a `Bytes` field. - pub fn set_bytes(&mut self, bytes: &[u8]) { - self.truncate_value_bytes(0); - self.0.extend(bytes); - } - /// Truncates the value bytes of the term. Value and field type stays the same. pub fn truncate_value_bytes(&mut self, len: usize) { - self.0.truncate(len + TERM_METADATA_LENGTH); + self.0.truncate(len + FIELD_ID_LENGTH); } /// The length of the bytes. pub fn len_bytes(&self) -> usize { - self.0.len() - TERM_METADATA_LENGTH + self.0.len() - FIELD_ID_LENGTH } - /// Appends value bytes to the Term. + /// Appends bytes to the Term. /// /// This function returns the segment that has just been added. #[inline] - pub fn append_bytes(&mut self, bytes: &[u8]) -> &mut [u8] { - let len_before = self.0.len(); + pub fn append_bytes(&mut self, bytes: &[u8]) { self.0.extend_from_slice(bytes); - &mut self.0[len_before..] - } -} - -impl IndexingTerm -where - B: AsRef<[u8]>, -{ - /// Wraps a object holding bytes - pub fn wrap(data: B) -> IndexingTerm { - IndexingTerm(data) - } - - /// Returns the field. - pub fn field(&self) -> Field { - let field_id_bytes: [u8; 4] = (&self.0.as_ref()[..4]).try_into().unwrap(); - Field::from_field_id(u32::from_be_bytes(field_id_bytes)) } /// Returns the serialized representation of Term. - /// This includes field_id, value type and value. - /// - /// Do NOT rely on this byte representation in the index. - /// This value is likely to change in the future. + /// This includes field_id, value bytes #[inline] - pub fn serialized_term(&self) -> &[u8] { + pub fn serialized_for_hashmap(&self) -> &[u8] { self.0.as_ref() } } + +pub fn get_field_from_indexing_term(bytes: &[u8]) -> Field { + let field_id_bytes: [u8; 4] = bytes[..4].try_into().unwrap(); + Field::from_field_id(u32::from_be_bytes(field_id_bytes)) +} diff --git a/src/schema/term.rs b/src/schema/term.rs index 05d0935d5..c5471ec9a 100644 --- a/src/schema/term.rs +++ b/src/schema/term.rs @@ -1,4 +1,4 @@ -use std::hash::{Hash, Hasher}; +use std::hash::Hash; use std::net::Ipv6Addr; use std::{fmt, str}; @@ -18,38 +18,40 @@ use crate::DateTime; /// 4 bytes are the field id, and the last byte is the type. /// /// The serialized value `ValueBytes` is considered everything after the 4 first bytes (term id). -#[derive(Clone)] -pub struct Term>(B) -where - B: AsRef<[u8]>; +#[derive(Clone, Hash, PartialEq, Ord, PartialOrd, Eq)] +pub struct Term(Vec); /// The number of bytes used as metadata by `Term`. const TERM_METADATA_LENGTH: usize = 5; impl Term { - /// Create a new Term with a buffer with a given capacity. - pub fn with_capacity(capacity: usize) -> Term { - let mut data = Vec::with_capacity(TERM_METADATA_LENGTH + capacity); + /// Create a new Term + pub fn new() -> Term { + let mut data = Vec::with_capacity(TERM_METADATA_LENGTH + 32); data.resize(TERM_METADATA_LENGTH, 0u8); Term(data) } pub(crate) fn with_type_and_field(typ: Type, field: Field) -> Term { - let mut term = Self::with_capacity(8); - term.set_field_and_type(field, typ); - term + Self::with_bytes_and_field_and_payload(typ, field, &[]) } fn with_bytes_and_field_and_payload(typ: Type, field: Field, bytes: &[u8]) -> Term { - let mut term = Self::with_capacity(bytes.len()); + let mut term = Self::new(); term.set_field_and_type(field, typ); term.0.extend_from_slice(bytes); term } + /// Sets a fast value in the term. + /// + /// fast values are converted to u64 and then serialized using (8-byte) BigEndian + /// representation. + /// The use of BigEndian has the benefit of preserving + /// the natural order of the values. fn from_fast_value(field: Field, val: &T) -> Term { let mut term = Self::with_type_and_field(T::to_type(), field); - term.set_u64(val.to_u64()); + term.set_bytes(val.to_u64().to_be_bytes().as_ref()); term } @@ -71,7 +73,7 @@ impl Term { /// Builds a term given a field, and a `Ipv6Addr`-value pub fn from_field_ip_addr(field: Field, ip_addr: Ipv6Addr) -> Term { let mut term = Self::with_type_and_field(Type::IpAddr, field); - term.set_ip_addr(ip_addr); + term.set_bytes(ip_addr.to_u128().to_be_bytes().as_ref()); term } @@ -122,40 +124,6 @@ impl Term { self.0[4] = typ.to_code(); } - /// Sets a u64 value in the term. - /// - /// U64 are serialized using (8-byte) BigEndian - /// representation. - /// The use of BigEndian has the benefit of preserving - /// the natural order of the values. - pub fn set_u64(&mut self, val: u64) { - self.set_fast_value(val); - } - - /// Sets a `i64` value in the term. - pub fn set_i64(&mut self, val: i64) { - self.set_fast_value(val); - } - - /// Sets a `DateTime` value in the term. - pub fn set_date(&mut self, date: DateTime) { - self.set_fast_value(date); - } - - /// Sets a `f64` value in the term. - pub fn set_f64(&mut self, val: f64) { - self.set_fast_value(val); - } - - /// Sets a `bool` value in the term. - pub fn set_bool(&mut self, val: bool) { - self.set_fast_value(val); - } - - fn set_fast_value(&mut self, val: T) { - self.set_bytes(val.to_u64().to_be_bytes().as_ref()); - } - /// Append a type marker + fast value to a term. /// This is used in JSON type to append a fast value after the path. /// @@ -181,13 +149,8 @@ impl Term { self.0.extend(val.as_bytes().as_ref()); } - /// Sets a `Ipv6Addr` value in the term. - pub fn set_ip_addr(&mut self, val: Ipv6Addr) { - self.set_bytes(val.to_u128().to_be_bytes().as_ref()); - } - /// Sets the value of a `Bytes` field. - pub fn set_bytes(&mut self, bytes: &[u8]) { + fn set_bytes(&mut self, bytes: &[u8]) { self.truncate_value_bytes(0); self.0.extend(bytes); } @@ -206,28 +169,12 @@ impl Term { self.0.extend_from_slice(bytes); &mut self.0[len_before..] } -} - -impl Term -where - B: AsRef<[u8]>, -{ - /// Wraps a object holding bytes - pub fn wrap(data: B) -> Term { - Term(data) - } /// Return the type of the term. pub fn typ(&self) -> Type { self.value().typ() } - /// Returns the field. - pub fn field(&self) -> Field { - let field_id_bytes: [u8; 4] = (&self.0.as_ref()[..4]).try_into().unwrap(); - Field::from_field_id(u32::from_be_bytes(field_id_bytes)) - } - /// Returns the serialized representation of the value. /// (this does neither include the field id nor the value type.) /// @@ -235,13 +182,7 @@ where /// If the term is a u64, its value is encoded according /// to `byteorder::BigEndian`. pub(crate) fn serialized_value_bytes(&self) -> &[u8] { - &self.0.as_ref()[TERM_METADATA_LENGTH..] - } - - /// Returns the value of the term. - /// address or JSON path + value. (this does not include the field.) - pub fn value(&self) -> ValueBytes<&[u8]> { - ValueBytes::wrap(&self.0.as_ref()[4..]) + &self.0[TERM_METADATA_LENGTH..] } /// Returns the serialized representation of Term. @@ -250,9 +191,22 @@ where /// Do NOT rely on this byte representation in the index. /// This value is likely to change in the future. #[inline] + #[cfg(test)] pub fn serialized_term(&self) -> &[u8] { self.0.as_ref() } + + /// Returns the field. + pub fn field(&self) -> Field { + let field_id_bytes: [u8; 4] = (&self.0[..4]).try_into().unwrap(); + Field::from_field_id(u32::from_be_bytes(field_id_bytes)) + } + + /// Returns the value of the term. + /// address or JSON path + value. (this does not include the field.) + pub fn value(&self) -> ValueBytes<&[u8]> { + ValueBytes::wrap(&self.0[4..]) + } } /// ValueBytes represents a serialized value. @@ -268,12 +222,10 @@ where /// The nested ValueBytes in JSON is never of type JSON. (there's no recursion) #[derive(Clone)] pub struct ValueBytes(B) -where - B: AsRef<[u8]>; +where B: AsRef<[u8]>; impl ValueBytes -where - B: AsRef<[u8]>, +where B: AsRef<[u8]> { /// Wraps a object holding bytes pub fn wrap(data: B) -> ValueBytes { @@ -285,18 +237,10 @@ where } /// Return the type of the term. - pub fn typ(&self) -> Type { + pub(crate) fn typ(&self) -> Type { Type::from_code(self.typ_code()).expect("The term has an invalid type code") } - /// Returns the `u64` value stored in a term. - /// - /// Returns `None` if the term is not of the u64 type, or if the term byte representation - /// is invalid. - pub fn as_u64(&self) -> Option { - self.get_fast_type::() - } - fn get_fast_type(&self) -> Option { if self.typ() != T::to_type() { return None; @@ -306,38 +250,6 @@ where Some(T::from_u64(value_u64)) } - /// Returns the `i64` value stored in a term. - /// - /// Returns `None` if the term is not of the i64 type, or if the term byte representation - /// is invalid. - pub fn as_i64(&self) -> Option { - self.get_fast_type::() - } - - /// Returns the `f64` value stored in a term. - /// - /// Returns `None` if the term is not of the f64 type, or if the term byte representation - /// is invalid. - pub fn as_f64(&self) -> Option { - self.get_fast_type::() - } - - /// Returns the `bool` value stored in a term. - /// - /// Returns `None` if the term is not of the bool type, or if the term byte representation - /// is invalid. - pub fn as_bool(&self) -> Option { - self.get_fast_type::() - } - - /// Returns the `Date` value stored in a term. - /// - /// Returns `None` if the term is not of the Date type, or if the term byte representation - /// is invalid. - pub fn as_date(&self) -> Option { - self.get_fast_type::() - } - /// Returns the text associated with the term. /// /// Returns `None` if the field is not of string type @@ -353,7 +265,7 @@ where /// /// Returns `None` if the field is not of facet type /// or if the bytes are not valid utf-8. - pub fn as_facet(&self) -> Option { + pub(crate) fn as_facet(&self) -> Option { if self.typ() != Type::Facet { return None; } @@ -364,7 +276,7 @@ where /// Returns the bytes associated with the term. /// /// Returns `None` if the field is not of bytes type. - pub fn as_bytes(&self) -> Option<&[u8]> { + pub(crate) fn as_bytes(&self) -> Option<&[u8]> { if self.typ() != Type::Bytes { return None; } @@ -372,7 +284,7 @@ where } /// Returns a `Ipv6Addr` value from the term. - pub fn as_ip_addr(&self) -> Option { + pub(crate) fn as_ip_addr(&self) -> Option { if self.typ() != Type::IpAddr { return None; } @@ -380,15 +292,6 @@ where Some(Ipv6Addr::from_u128(ip_u128)) } - /// Returns the json path type. - /// - /// Returns `None` if the value is not JSON. - pub fn json_path_type(&self) -> Option { - let json_value_bytes = self.as_json_value_bytes()?; - - Some(json_value_bytes.typ()) - } - /// Returns the json path bytes (including the JSON_END_OF_PATH byte), /// and the encoded ValueBytes after the json path. /// @@ -405,18 +308,6 @@ where Some((json_path_bytes, ValueBytes::wrap(term))) } - /// Returns the encoded ValueBytes after the json path. - /// - /// Returns `None` if the value is not JSON. - pub(crate) fn as_json_value_bytes(&self) -> Option> { - if self.typ() != Type::Json { - return None; - } - let bytes = self.value_bytes(); - let pos = bytes.iter().cloned().position(|b| b == JSON_END_OF_PATH)?; - Some(ValueBytes::wrap(&bytes[pos + 1..])) - } - /// Returns the serialized value of ValueBytes without the type. fn value_bytes(&self) -> &[u8] { &self.0.as_ref()[1..] @@ -439,20 +330,20 @@ where write_opt(f, s)?; } Type::U64 => { - write_opt(f, self.as_u64())?; + write_opt(f, self.get_fast_type::())?; } Type::I64 => { - write_opt(f, self.as_i64())?; + write_opt(f, self.get_fast_type::())?; } Type::F64 => { - write_opt(f, self.as_f64())?; + write_opt(f, self.get_fast_type::())?; } Type::Bool => { - write_opt(f, self.as_bool())?; + write_opt(f, self.get_fast_type::())?; } // TODO pretty print these types too. Type::Date => { - write_opt(f, self.as_date())?; + write_opt(f, self.get_fast_type::())?; } Type::Facet => { write_opt(f, self.as_facet())?; @@ -478,44 +369,6 @@ where } } -impl Ord for Term -where - B: AsRef<[u8]>, -{ - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.serialized_term().cmp(other.serialized_term()) - } -} - -impl PartialOrd for Term -where - B: AsRef<[u8]>, -{ - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl PartialEq for Term -where - B: AsRef<[u8]>, -{ - fn eq(&self, other: &Self) -> bool { - self.serialized_term() == other.serialized_term() - } -} - -impl Eq for Term where B: AsRef<[u8]> {} - -impl Hash for Term -where - B: AsRef<[u8]>, -{ - fn hash(&self, state: &mut H) { - self.0.as_ref().hash(state) - } -} - fn write_opt(f: &mut fmt::Formatter, val_opt: Option) -> fmt::Result { if let Some(val) = val_opt { write!(f, "{val:?}")?; @@ -523,14 +376,11 @@ fn write_opt(f: &mut fmt::Formatter, val_opt: Option) -> Ok(()) } -impl fmt::Debug for Term -where - B: AsRef<[u8]>, -{ +impl fmt::Debug for Term { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let field_id = self.field().field_id(); write!(f, "Term(field={field_id}, ")?; - let value_bytes = ValueBytes::wrap(&self.0.as_ref()[4..]); + let value_bytes = ValueBytes::wrap(&self.0[4..]); value_bytes.debug_value_bytes(f)?; write!(f, ")",)?; Ok(()) @@ -552,38 +402,4 @@ mod tests { assert_eq!(term.typ(), Type::Str); assert_eq!(term.value().as_str(), Some("test")) } - - /// Size (in bytes) of the buffer of a fast value (u64, i64, f64, or date) term. - /// + + - /// - /// - is a big endian encoded u32 field id - /// - 's most significant bit expresses whether the term is a json term or not - /// The remaining 7 bits are used to encode the type of the value. - /// If this is a JSON term, the type is the type of the leaf of the json. - /// - /// - is, if this is not the json term, a binary representation specific to the type. - /// If it is a JSON Term, then it is prepended with the path that leads to this leaf value. - const FAST_VALUE_TERM_LEN: usize = 4 + 1 + 8; - - #[test] - pub fn test_term_u64() { - let mut schema_builder = Schema::builder(); - let count_field = schema_builder.add_u64_field("count", INDEXED); - let term = Term::from_field_u64(count_field, 983u64); - assert_eq!(term.field(), count_field); - assert_eq!(term.typ(), Type::U64); - assert_eq!(term.serialized_term().len(), FAST_VALUE_TERM_LEN); - assert_eq!(term.value().as_u64(), Some(983u64)) - } - - #[test] - pub fn test_term_bool() { - let mut schema_builder = Schema::builder(); - let bool_field = schema_builder.add_bool_field("bool", INDEXED); - let term = Term::from_field_bool(bool_field, true); - assert_eq!(term.field(), bool_field); - assert_eq!(term.typ(), Type::Bool); - assert_eq!(term.serialized_term().len(), FAST_VALUE_TERM_LEN); - assert_eq!(term.value().as_bool(), Some(true)) - } }