rust/tantivy
1
0
Fork 0
mirror of https://github.com/quickwit-oss/tantivy.git synced 2026-06-03 09:00:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

rename ip addr, use buffer

Browse Source
This commit is contained in:
Pascal Seitz
2022-10-06 13:13:05 +08:00
parent 5d6602a8d9
commit 0b86658389
8 changed files with 66 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/fastfield/multivalued/writer.rs
View File

@@ -318,10 +318,10 @@ impl MultiValueU128FastFieldWriter {
if field_value.field == self.field {
let value = field_value.value();
let ip_addr = value
.as_ip()
.as_ip_addr()
.unwrap_or_else(|| panic!("expected and ip, but got {:?}", value));
let value = ip_addr.to_u128();
self.add_val(value);
let ip_addr_u128 = ip_addr.to_u128();
self.add_val(ip_addr_u128);
}
}
}

4
src/fastfield/writer.rs
View File

@@ -330,7 +330,7 @@ impl U128FastFieldWriter {
match doc.get_first(self.field) {
Some(v) => {
let ip_addr = v
.as_ip()
.as_ip_addr()
.unwrap_or_else(|| panic!("expected and ip, but got {:?}", v));
let value = ip_addr.to_u128();
@@ -359,7 +359,7 @@ impl U128FastFieldWriter {
};
serialize_u128(iter_gen, self.val_count as u64, field_write)?;
} else {
let iter_gen = || (0..self.val_count).map(|idx| self.vals[idx as usize]);
let iter_gen = || self.vals.iter().cloned();
serialize_u128(iter_gen, self.val_count as u64, field_write)?;
}

13
src/indexer/merger.rs
View File

@@ -6,6 +6,7 @@ use fastfield_codecs::{serialize_u128, VecColumn};
use itertools::Itertools;
use measure_time::debug_time;
use super::flat_map_with_buffer::FlatMapWithBufferIter;
use super::sorted_doc_id_multivalue_column::RemappedDocIdMultiValueIndexColumn;
use crate::core::{Segment, SegmentReader};
use crate::docset::{DocSet, TERMINATED};
@@ -356,12 +357,12 @@ impl IndexMerger {
.collect::<Vec<_>>();
let iter_gen = || {
doc_id_mapping.iter_old_doc_addrs().flat_map(|doc_addr| {
let fast_field_reader = &fast_field_readers[doc_addr.segment_ord as usize];
let mut out = vec![];
fast_field_reader.get_vals(doc_addr.doc_id, &mut out);
out.into_iter()
})
doc_id_mapping
.iter_old_doc_addrs()
.flat_map_with_buffer(|doc_addr, buffer| {
let fast_field_reader = &fast_field_readers[doc_addr.segment_ord as usize];
fast_field_reader.get_vals(doc_addr.doc_id, buffer);
})
};
let field_write = fast_field_serializer.get_field_writer(field, 1);
serialize_u128(iter_gen, doc_id_mapping.len() as u64, field_write)?;

6
src/query/query_parser/query_parser.rs
View File

@@ -400,7 +400,11 @@ impl QueryParser {
let bytes = base64::decode(phrase).map_err(QueryParserError::ExpectedBase64)?;
Ok(Term::from_field_bytes(field, &bytes))
}
FieldType::IpAddr(_) => Ok(Term::from_field_text(field, phrase)),
FieldType::IpAddr(_) => {
return Err(QueryParserError::UnsupportedQuery(
"Range query are not supported on IpAddr field.".to_string(),
));
}
}
}

2
src/schema/field_entry.rs
View File

@@ -61,7 +61,7 @@ impl FieldEntry {
Self::new(field_name, FieldType::Date(date_options))
}
/// Creates a new ip field entry.
/// Creates a new ip address field entry.
pub fn new_ip_addr(field_name: String, ip_options: IpAddrOptions) -> FieldEntry {
Self::new(field_name, FieldType::IpAddr(ip_options))
}

70
src/schema/field_type.rs
View File

@@ -228,7 +228,7 @@ impl FieldType {
| FieldType::F64(ref int_options)
| FieldType::Bool(ref int_options) => int_options.is_fast(),
FieldType::Date(ref date_options) => date_options.is_fast(),
FieldType::IpAddr(ref options) => options.is_fast(),
FieldType::IpAddr(ref ip_addr_options) => ip_addr_options.is_fast(),
FieldType::Facet(_) => true,
FieldType::JsonObject(_) => false,
}
@@ -325,45 +325,43 @@ impl FieldType {
/// target field is a `Str`, this method will return an Error.
pub fn value_from_json(&self, json: JsonValue) -> Result<Value, ValueParsingError> {
match json {
JsonValue::String(field_text) => {
match self {
FieldType::Date(_) => {
let dt_with_fixed_tz = OffsetDateTime::parse(&field_text, &Rfc3339)
.map_err(|_err| ValueParsingError::TypeError {
JsonValue::String(field_text) => match self {
FieldType::Date(_) => {
let dt_with_fixed_tz =
OffsetDateTime::parse(&field_text, &Rfc3339).map_err(|_err| {
ValueParsingError::TypeError {
expected: "rfc3339 format",
json: JsonValue::String(field_text),
})?;
Ok(DateTime::from_utc(dt_with_fixed_tz).into())
}
FieldType::Str(_) => Ok(Value::Str(field_text)),
FieldType::U64(_) | FieldType::I64(_) | FieldType::F64(_) => {
Err(ValueParsingError::TypeError {
expected: "an integer",
json: JsonValue::String(field_text),
})
}
FieldType::Bool(_) => Err(ValueParsingError::TypeError {
expected: "a boolean",
json: JsonValue::String(field_text),
}),
FieldType::Facet(_) => Ok(Value::Facet(Facet::from(&field_text))),
FieldType::Bytes(_) => base64::decode(&field_text)
.map(Value::Bytes)
.map_err(|_| ValueParsingError::InvalidBase64 { base64: field_text }),
FieldType::JsonObject(_) => Err(ValueParsingError::TypeError {
expected: "a json object",
json: JsonValue::String(field_text),
}),
FieldType::IpAddr(_) => {
Ok(Value::Ip(IpAddr::from_str(&field_text).map_err(|err| {
ValueParsingError::ParseError {
error: err.to_string(),
json: JsonValue::String(field_text),
}
})?))
}
})?;
Ok(DateTime::from_utc(dt_with_fixed_tz).into())
}
}
FieldType::Str(_) => Ok(Value::Str(field_text)),
FieldType::U64(_) | FieldType::I64(_) | FieldType::F64(_) => {
Err(ValueParsingError::TypeError {
expected: "an integer",
json: JsonValue::String(field_text),
})
}
FieldType::Bool(_) => Err(ValueParsingError::TypeError {
expected: "a boolean",
json: JsonValue::String(field_text),
}),
FieldType::Facet(_) => Ok(Value::Facet(Facet::from(&field_text))),
FieldType::Bytes(_) => base64::decode(&field_text)
.map(Value::Bytes)
.map_err(|_| ValueParsingError::InvalidBase64 { base64: field_text }),
FieldType::JsonObject(_) => Err(ValueParsingError::TypeError {
expected: "a json object",
json: JsonValue::String(field_text),
}),
FieldType::IpAddr(_) => Ok(Value::IpAddr(IpAddr::from_str(&field_text).map_err(
|err| ValueParsingError::ParseError {
error: err.to_string(),
json: JsonValue::String(field_text),
},
)?)),
},
JsonValue::Number(field_val_num) => match self {
FieldType::I64(_) | FieldType::Date(_) => {
if let Some(field_val_i64) = field_val_num.as_i64() {

4
src/schema/schema.rs
View File

@@ -146,9 +146,7 @@ impl SchemaBuilder {
}
/// Adds a ip field.
/// Returns the associated field handle
/// Internally, Tantivy simply stores ips as u64,
/// while the user supplies IpAddr values for convenience.
/// Returns the associated field handle.
///
/// # Caution
///

22
src/schema/value.rs
View File

@@ -34,14 +34,16 @@ pub enum Value {
/// Json object value.
JsonObject(serde_json::Map<String, serde_json::Value>),
/// Ip
Ip(IpAddr),
IpAddr(IpAddr),
}
impl Eq for Value {}
impl Serialize for Value {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where S: Serializer {
where
S: Serializer,
{
match *self {
Value::Str(ref v) => serializer.serialize_str(v),
Value::PreTokStr(ref v) => v.serialize(serializer),
@@ -53,14 +55,16 @@ impl Serialize for Value {
Value::Facet(ref facet) => facet.serialize(serializer),
Value::Bytes(ref bytes) => serializer.serialize_bytes(bytes),
Value::JsonObject(ref obj) => obj.serialize(serializer),
Value::Ip(ref obj) => obj.serialize(serializer), // TODO check serialization
Value::IpAddr(ref obj) => obj.serialize(serializer), // TODO check serialization
}
}
}
impl<'de> Deserialize<'de> for Value {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where D: Deserializer<'de> {
where
D: Deserializer<'de>,
{
struct ValueVisitor;
impl<'de> Visitor<'de> for ValueVisitor {
@@ -208,8 +212,8 @@ impl Value {
/// Returns the ip addr, provided the value is of the `Ip` type.
/// (Returns None if the value is not of the `Ip` type)
pub fn as_ip(&self) -> Option<IpAddr> {
if let Value::Ip(val) = self {
pub fn as_ip_addr(&self) -> Option<IpAddr> {
if let Value::IpAddr(val) = self {
Some(*val)
} else {
None
@@ -225,7 +229,7 @@ impl From<String> for Value {
impl From<IpAddr> for Value {
fn from(v: IpAddr) -> Value {
Value::Ip(v)
Value::IpAddr(v)
}
}
@@ -389,7 +393,7 @@ mod binary_serialize {
serde_json::to_writer(writer, &map)?;
Ok(())
}
Value::Ip(ref ip) => {
Value::IpAddr(ref ip) => {
IP_CODE.serialize(writer)?;
ip.to_string().serialize(writer) // TODO Check best format
}
@@ -465,7 +469,7 @@ mod binary_serialize {
}
IP_CODE => {
let text = String::deserialize(reader)?;
Ok(Value::Ip(IpAddr::from_str(&text).map_err(|err| {
Ok(Value::IpAddr(IpAddr::from_str(&text).map_err(|err| {
io::Error::new(ErrorKind::Other, err.to_string())
})?))
}