mirror of
https://github.com/quickwit-oss/tantivy.git
synced 2026-06-04 09:30:42 +00:00
add support for ip range query on multivalue fastfields
This commit is contained in:
Pascal Seitz
@@ -113,7 +113,7 @@ mod tests {
|
||||
|
||||
b.iter(|| {
|
||||
let mut positions = Vec::new();
|
||||
column.get_positions_for_value_range(
|
||||
column.get_docids_for_value_range(
|
||||
major_item..=major_item,
|
||||
0..data.len() as u32,
|
||||
&mut positions,
|
||||
@@ -129,7 +129,7 @@ mod tests {
|
||||
|
||||
b.iter(|| {
|
||||
let mut positions = Vec::new();
|
||||
column.get_positions_for_value_range(
|
||||
column.get_docids_for_value_range(
|
||||
minor_item..=minor_item,
|
||||
0..data.len() as u32,
|
||||
&mut positions,
|
||||
@@ -145,11 +145,7 @@ mod tests {
|
||||
|
||||
b.iter(|| {
|
||||
let mut positions = Vec::new();
|
||||
column.get_positions_for_value_range(
|
||||
0..=u128::MAX,
|
||||
0..data.len() as u32,
|
||||
&mut positions,
|
||||
);
|
||||
column.get_docids_for_value_range(0..=u128::MAX, 0..data.len() as u32, &mut positions);
|
||||
positions
|
||||
});
|
||||
}
|
||||
|
||||
@@ -35,7 +35,7 @@ pub trait Column<T: PartialOrd = u64>: Send + Sync {
|
||||
///
|
||||
/// Note that position == docid for single value fast fields
|
||||
#[inline]
|
||||
fn get_positions_for_value_range(
|
||||
fn get_docids_for_value_range(
|
||||
&self,
|
||||
value_range: RangeInclusive<T>,
|
||||
doc_id_range: Range<u32>,
|
||||
@@ -70,6 +70,11 @@ pub trait Column<T: PartialOrd = u64>: Send + Sync {
|
||||
/// The number of values in the column.
|
||||
fn num_vals(&self) -> u32;
|
||||
|
||||
/// The number of docs in the column. For single value columns this equals num_vals.
|
||||
fn num_docs(&self) -> u32 {
|
||||
self.num_vals()
|
||||
}
|
||||
|
||||
/// Returns a iterator over the data
|
||||
fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = T> + 'a> {
|
||||
Box::new((0..self.num_vals()).map(|idx| self.get_val(idx)))
|
||||
@@ -222,13 +227,13 @@ where
|
||||
)
|
||||
}
|
||||
|
||||
fn get_positions_for_value_range(
|
||||
fn get_docids_for_value_range(
|
||||
&self,
|
||||
range: RangeInclusive<Output>,
|
||||
doc_id_range: Range<u32>,
|
||||
positions: &mut Vec<u32>,
|
||||
) {
|
||||
self.from_column.get_positions_for_value_range(
|
||||
self.from_column.get_docids_for_value_range(
|
||||
self.monotonic_mapping.inverse(range.start().clone())
|
||||
..=self.monotonic_mapping.inverse(range.end().clone()),
|
||||
doc_id_range,
|
||||
|
||||
@@ -306,13 +306,13 @@ impl Column<u128> for CompactSpaceDecompressor {
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn get_positions_for_value_range(
|
||||
fn get_docids_for_value_range(
|
||||
&self,
|
||||
value_range: RangeInclusive<u128>,
|
||||
doc_id_range: Range<u32>,
|
||||
positions_range: Range<u32>,
|
||||
positions: &mut Vec<u32>,
|
||||
) {
|
||||
self.get_positions_for_value_range(value_range, doc_id_range, positions)
|
||||
self.get_positions_for_value_range(value_range, positions_range, positions)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -351,13 +351,13 @@ impl CompactSpaceDecompressor {
|
||||
pub fn get_positions_for_value_range(
|
||||
&self,
|
||||
value_range: RangeInclusive<u128>,
|
||||
doc_id_range: Range<u32>,
|
||||
position_range: Range<u32>,
|
||||
positions: &mut Vec<u32>,
|
||||
) {
|
||||
if value_range.start() > value_range.end() {
|
||||
return;
|
||||
}
|
||||
let doc_id_range = doc_id_range.start..doc_id_range.end.min(self.num_vals());
|
||||
let position_range = position_range.start..position_range.end.min(self.num_vals());
|
||||
let from_value = *value_range.start();
|
||||
let to_value = *value_range.end();
|
||||
assert!(to_value >= from_value);
|
||||
@@ -390,10 +390,10 @@ impl CompactSpaceDecompressor {
|
||||
|
||||
let range = compact_from..=compact_to;
|
||||
|
||||
let scan_num_docs = doc_id_range.end - doc_id_range.start;
|
||||
let scan_num_docs = position_range.end - position_range.start;
|
||||
|
||||
let step_size = 4;
|
||||
let cutoff = doc_id_range.start + scan_num_docs - scan_num_docs % step_size;
|
||||
let cutoff = position_range.start + scan_num_docs - scan_num_docs % step_size;
|
||||
|
||||
let mut push_if_in_range = |idx, val| {
|
||||
if range.contains(&val) {
|
||||
@@ -402,7 +402,7 @@ impl CompactSpaceDecompressor {
|
||||
};
|
||||
let get_val = |idx| self.params.bit_unpacker.get(idx, &self.data);
|
||||
// unrolled loop
|
||||
for idx in (doc_id_range.start..cutoff).step_by(step_size as usize) {
|
||||
for idx in (position_range.start..cutoff).step_by(step_size as usize) {
|
||||
let idx1 = idx;
|
||||
let idx2 = idx + 1;
|
||||
let idx3 = idx + 2;
|
||||
@@ -418,7 +418,7 @@ impl CompactSpaceDecompressor {
|
||||
}
|
||||
|
||||
// handle rest
|
||||
for idx in cutoff..doc_id_range.end {
|
||||
for idx in cutoff..position_range.end {
|
||||
push_if_in_range(idx, get_val(idx as u32));
|
||||
}
|
||||
}
|
||||
@@ -708,7 +708,7 @@ mod tests {
|
||||
doc_id_range: Range<u32>,
|
||||
) -> Vec<u32> {
|
||||
let mut positions = Vec::new();
|
||||
column.get_positions_for_value_range(value_range, doc_id_range, &mut positions);
|
||||
column.get_docids_for_value_range(value_range, doc_id_range, &mut positions);
|
||||
positions
|
||||
}
|
||||
|
||||
|
||||
@@ -255,7 +255,7 @@ mod tests {
|
||||
.map(|(pos, _)| pos as u32)
|
||||
.collect();
|
||||
let mut positions = Vec::new();
|
||||
reader.get_positions_for_value_range(
|
||||
reader.get_docids_for_value_range(
|
||||
data[test_rand_idx]..=data[test_rand_idx],
|
||||
0..data.len() as u32,
|
||||
&mut positions,
|
||||
|
||||
@@ -119,7 +119,7 @@ fn bench_ip() {
|
||||
for value in dataset.iter().take(1110).skip(1100).cloned() {
|
||||
doc_values.clear();
|
||||
print_time!("get range");
|
||||
decompressor.get_positions_for_value_range(
|
||||
decompressor.get_docids_for_value_range(
|
||||
value..=value,
|
||||
0..decompressor.num_vals(),
|
||||
&mut doc_values,
|
||||
|
||||
@@ -26,6 +26,21 @@ impl MultiValueIndex {
|
||||
start..end
|
||||
}
|
||||
|
||||
/// Returns `[start, end)`, such that the values associated with
|
||||
/// the given documents are `start..end`.
|
||||
///
|
||||
/// The passed end range is allowed to be out of bounds.
|
||||
#[inline]
|
||||
pub(crate) fn docid_range_to_position_range(&self, range: Range<DocId>) -> Range<u32> {
|
||||
let end_docid = range.end.min(self.num_docs() - 1);
|
||||
let start_docid = range.start.min(end_docid);
|
||||
|
||||
let start = self.idx.get_val(start_docid) as u32;
|
||||
let end = self.idx.get_val(end_docid + 1) as u32;
|
||||
|
||||
start..end
|
||||
}
|
||||
|
||||
/// returns the num of values associated with a doc_id
|
||||
pub(crate) fn num_vals_for_doc(&self, doc: DocId) -> u32 {
|
||||
let range = self.range(doc);
|
||||
@@ -45,6 +60,7 @@ impl MultiValueIndex {
|
||||
}
|
||||
|
||||
/// Converts a list of positions of values in a 1:n index to the corresponding list of DocIds.
|
||||
/// Positions are converted inplace to docids.
|
||||
///
|
||||
/// Since there is no index for value pos -> docid, but docid -> value pos range, we scan the
|
||||
/// index.
|
||||
@@ -54,40 +70,52 @@ impl MultiValueIndex {
|
||||
///
|
||||
/// TODO: Instead of a linear scan we can employ a exponential search into binary search to
|
||||
/// match a docid to its value position.
|
||||
pub(crate) fn positions_to_docids(&self, docid_start: u32, positions: &[u32]) -> Vec<DocId> {
|
||||
let mut docs = vec![];
|
||||
let mut cur_doc = docid_start;
|
||||
pub(crate) fn positions_to_docids(&self, doc_id_range: Range<u32>, positions: &mut Vec<u32>) {
|
||||
let mut cur_doc = doc_id_range.start;
|
||||
let mut last_doc = None;
|
||||
|
||||
for pos in positions {
|
||||
let mut write_doc_pos = 0;
|
||||
for i in 0..positions.len() {
|
||||
let pos = positions[i];
|
||||
loop {
|
||||
let end = self.idx.get_val(cur_doc + 1) as u32;
|
||||
if end > *pos {
|
||||
if end > pos {
|
||||
// avoid duplicates
|
||||
if Some(cur_doc) == last_doc {
|
||||
break;
|
||||
}
|
||||
docs.push(cur_doc);
|
||||
positions[write_doc_pos] = cur_doc;
|
||||
write_doc_pos += 1;
|
||||
|
||||
last_doc = Some(cur_doc);
|
||||
break;
|
||||
}
|
||||
cur_doc += 1;
|
||||
}
|
||||
}
|
||||
|
||||
docs
|
||||
positions.truncate(write_doc_pos);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
use std::ops::Range;
|
||||
use std::sync::Arc;
|
||||
|
||||
use fastfield_codecs::IterColumn;
|
||||
|
||||
use crate::fastfield::MultiValueIndex;
|
||||
|
||||
fn index_to_pos_helper(
|
||||
index: &MultiValueIndex,
|
||||
doc_id_range: Range<u32>,
|
||||
positions: &[u32],
|
||||
) -> Vec<u32> {
|
||||
let mut positions = positions.to_vec();
|
||||
index.positions_to_docids(doc_id_range, &mut positions);
|
||||
positions
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_positions_to_docid() {
|
||||
let offsets = vec![0, 10, 12, 15, 22, 23]; // docid values are [0..10, 10..12, 12..15, etc.]
|
||||
@@ -97,14 +125,14 @@ mod tests {
|
||||
{
|
||||
let positions = vec![10u32, 11, 15, 20, 21, 22];
|
||||
|
||||
assert_eq!(index.positions_to_docids(0, &positions), vec![1, 3, 4]);
|
||||
assert_eq!(index.positions_to_docids(1, &positions), vec![1, 3, 4]);
|
||||
assert_eq!(index.positions_to_docids(0, &[9]), vec![0]);
|
||||
assert_eq!(index.positions_to_docids(1, &[10]), vec![1]);
|
||||
assert_eq!(index.positions_to_docids(1, &[11]), vec![1]);
|
||||
assert_eq!(index.positions_to_docids(2, &[12]), vec![2]);
|
||||
assert_eq!(index.positions_to_docids(2, &[12, 14]), vec![2]);
|
||||
assert_eq!(index.positions_to_docids(2, &[12, 14, 15]), vec![2, 3]);
|
||||
assert_eq!(index_to_pos_helper(&index, 0..5, &positions), vec![1, 3, 4]);
|
||||
assert_eq!(index_to_pos_helper(&index, 1..5, &positions), vec![1, 3, 4]);
|
||||
assert_eq!(index_to_pos_helper(&index, 0..5, &[9]), vec![0]);
|
||||
assert_eq!(index_to_pos_helper(&index, 1..5, &[10]), vec![1]);
|
||||
assert_eq!(index_to_pos_helper(&index, 1..5, &[11]), vec![1]);
|
||||
assert_eq!(index_to_pos_helper(&index, 2..5, &[12]), vec![2]);
|
||||
assert_eq!(index_to_pos_helper(&index, 2..5, &[12, 14]), vec![2]);
|
||||
assert_eq!(index_to_pos_helper(&index, 2..5, &[12, 14, 15]), vec![2, 3]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ use crate::DocId;
|
||||
/// The `vals_reader` will access the concatenated list of all
|
||||
/// values for all reader.
|
||||
/// The `idx_reader` associated, for each document, the index of its first value.
|
||||
/// Stores the start position for each document.
|
||||
#[derive(Clone)]
|
||||
pub struct MultiValuedFastFieldReader<Item: FastValue> {
|
||||
idx_reader: MultiValueIndex,
|
||||
@@ -138,19 +139,6 @@ impl<T: MonotonicallyMappableToU128> MultiValuedU128FastFieldReader<T> {
|
||||
self.get_vals_for_range(range, vals);
|
||||
}
|
||||
|
||||
/// Returns all docids which are in the provided value range
|
||||
pub fn get_positions_for_value_range(
|
||||
&self,
|
||||
value_range: RangeInclusive<T>,
|
||||
doc_id_range: Range<u32>,
|
||||
) -> Vec<DocId> {
|
||||
let mut positions = Vec::new();
|
||||
self.vals_reader
|
||||
.get_positions_for_value_range(value_range, doc_id_range, &mut positions);
|
||||
|
||||
self.idx_reader.positions_to_docids(0, &positions)
|
||||
}
|
||||
|
||||
/// Iterates over all elements in the fast field
|
||||
pub fn iter(&self) -> impl Iterator<Item = T> + '_ {
|
||||
self.vals_reader.iter()
|
||||
@@ -191,6 +179,44 @@ impl<T: MonotonicallyMappableToU128> MultiValuedU128FastFieldReader<T> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: MonotonicallyMappableToU128> Column<T> for MultiValuedU128FastFieldReader<T> {
|
||||
fn get_val(&self, _idx: u32) -> T {
|
||||
panic!("calling get_val on a multivalue field indicates a bug")
|
||||
}
|
||||
|
||||
fn min_value(&self) -> T {
|
||||
(self as &MultiValuedU128FastFieldReader<T>).min_value()
|
||||
}
|
||||
|
||||
fn max_value(&self) -> T {
|
||||
(self as &MultiValuedU128FastFieldReader<T>).max_value()
|
||||
}
|
||||
|
||||
fn num_vals(&self) -> u32 {
|
||||
self.total_num_vals() as u32
|
||||
}
|
||||
|
||||
fn num_docs(&self) -> u32 {
|
||||
self.get_index_reader().num_docs()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn get_docids_for_value_range(
|
||||
&self,
|
||||
value_range: RangeInclusive<T>,
|
||||
doc_id_range: Range<u32>,
|
||||
positions: &mut Vec<u32>,
|
||||
) {
|
||||
let position_range = self
|
||||
.get_index_reader()
|
||||
.docid_range_to_position_range(doc_id_range.clone());
|
||||
self.vals_reader
|
||||
.get_docids_for_value_range(value_range, position_range, positions);
|
||||
|
||||
self.idx_reader.positions_to_docids(doc_id_range, positions);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
|
||||
@@ -47,12 +47,23 @@ impl Weight for IPFastFieldRangeWeight {
|
||||
let value_range = bound_to_value_range(
|
||||
&self.left_bound,
|
||||
&self.right_bound,
|
||||
ip_addr_fast_field.as_ref(),
|
||||
ip_addr_fast_field.min_value(),
|
||||
ip_addr_fast_field.max_value(),
|
||||
);
|
||||
let docset = IpRangeDocSet::new(value_range, ip_addr_fast_field);
|
||||
let docset = IpRangeDocSet::new(value_range, ip_addr_fast_field, false);
|
||||
Ok(Box::new(ConstScorer::new(docset, boost)))
|
||||
}
|
||||
Cardinality::MultiValues => {
|
||||
let ip_addr_fast_field = reader.fast_fields().ip_addrs(self.field)?;
|
||||
let value_range = bound_to_value_range(
|
||||
&self.left_bound,
|
||||
&self.right_bound,
|
||||
ip_addr_fast_field.min_value(),
|
||||
ip_addr_fast_field.max_value(),
|
||||
);
|
||||
let docset = IpRangeDocSet::new(value_range, Arc::new(ip_addr_fast_field), true);
|
||||
Ok(Box::new(ConstScorer::new(docset, boost)))
|
||||
}
|
||||
Cardinality::MultiValues => unimplemented!(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,18 +84,19 @@ impl Weight for IPFastFieldRangeWeight {
|
||||
fn bound_to_value_range(
|
||||
left_bound: &Bound<Ipv6Addr>,
|
||||
right_bound: &Bound<Ipv6Addr>,
|
||||
column: &dyn Column<Ipv6Addr>,
|
||||
min_value: Ipv6Addr,
|
||||
max_value: Ipv6Addr,
|
||||
) -> RangeInclusive<Ipv6Addr> {
|
||||
let start_value = match left_bound {
|
||||
Bound::Included(ip_addr) => *ip_addr,
|
||||
Bound::Excluded(ip_addr) => Ipv6Addr::from(ip_addr.to_u128() + 1),
|
||||
Bound::Unbounded => column.min_value(),
|
||||
Bound::Unbounded => min_value,
|
||||
};
|
||||
|
||||
let end_value = match right_bound {
|
||||
Bound::Included(ip_addr) => *ip_addr,
|
||||
Bound::Excluded(ip_addr) => Ipv6Addr::from(ip_addr.to_u128() - 1),
|
||||
Bound::Unbounded => column.max_value(),
|
||||
Bound::Unbounded => max_value,
|
||||
};
|
||||
start_value..=end_value
|
||||
}
|
||||
@@ -109,13 +121,14 @@ impl VecCursor {
|
||||
fn current(&self) -> Option<u32> {
|
||||
self.docs.get(self.current_pos).map(|el| *el as u32)
|
||||
}
|
||||
|
||||
fn get_cleared_data(&mut self) -> &mut Vec<u32> {
|
||||
self.docs.clear();
|
||||
self.current_pos = 0;
|
||||
&mut self.docs
|
||||
}
|
||||
|
||||
fn last_value(&self) -> Option<u32> {
|
||||
self.docs.iter().last().cloned()
|
||||
}
|
||||
fn is_empty(&self) -> bool {
|
||||
self.current_pos >= self.docs.len()
|
||||
}
|
||||
@@ -139,21 +152,25 @@ struct IpRangeDocSet {
|
||||
/// Current batch of loaded docs.
|
||||
loaded_docs: VecCursor,
|
||||
last_seek_pos_opt: Option<u32>,
|
||||
/// If fast field is multivalue.
|
||||
is_multivalue: bool,
|
||||
}
|
||||
|
||||
const DEFALT_FETCH_HORIZON: u32 = 128;
|
||||
const DEFAULT_FETCH_HORIZON: u32 = 128;
|
||||
impl IpRangeDocSet {
|
||||
fn new(
|
||||
value_range: RangeInclusive<Ipv6Addr>,
|
||||
ip_addr_fast_field: Arc<dyn Column<Ipv6Addr>>,
|
||||
is_multivalue: bool,
|
||||
) -> Self {
|
||||
let mut ip_range_docset = Self {
|
||||
value_range,
|
||||
ip_addr_fast_field,
|
||||
loaded_docs: VecCursor::new(),
|
||||
next_fetch_start: 0,
|
||||
fetch_horizon: DEFALT_FETCH_HORIZON,
|
||||
fetch_horizon: DEFAULT_FETCH_HORIZON,
|
||||
last_seek_pos_opt: None,
|
||||
is_multivalue,
|
||||
};
|
||||
ip_range_docset.reset_fetch_range();
|
||||
ip_range_docset.fetch_block();
|
||||
@@ -161,7 +178,7 @@ impl IpRangeDocSet {
|
||||
}
|
||||
|
||||
fn reset_fetch_range(&mut self) {
|
||||
self.fetch_horizon = DEFALT_FETCH_HORIZON;
|
||||
self.fetch_horizon = DEFAULT_FETCH_HORIZON;
|
||||
}
|
||||
|
||||
/// Returns true if more data could be fetched
|
||||
@@ -190,19 +207,30 @@ impl IpRangeDocSet {
|
||||
fn fetch_horizon(&mut self, horizon: u32) -> bool {
|
||||
let mut finished_to_end = false;
|
||||
|
||||
let limit = self.ip_addr_fast_field.num_vals();
|
||||
let limit = self.ip_addr_fast_field.num_docs();
|
||||
let mut end = self.next_fetch_start + horizon;
|
||||
if end >= limit {
|
||||
end = limit;
|
||||
finished_to_end = true;
|
||||
}
|
||||
|
||||
let data = self.loaded_docs.get_cleared_data();
|
||||
self.ip_addr_fast_field.get_positions_for_value_range(
|
||||
let last_loaded_docs_val = self
|
||||
.is_multivalue
|
||||
.then(|| self.loaded_docs.last_value())
|
||||
.flatten();
|
||||
|
||||
let loaded_docs_data = self.loaded_docs.get_cleared_data();
|
||||
self.ip_addr_fast_field.get_docids_for_value_range(
|
||||
self.value_range.clone(),
|
||||
self.next_fetch_start..end,
|
||||
data,
|
||||
loaded_docs_data,
|
||||
);
|
||||
// In case of multivalues, we may have an overlap of the same docid between fetching blocks
|
||||
if let Some(last_value) = last_loaded_docs_val {
|
||||
while self.loaded_docs.current() == Some(last_value) {
|
||||
self.loaded_docs.next();
|
||||
}
|
||||
}
|
||||
self.next_fetch_start = end;
|
||||
finished_to_end
|
||||
}
|
||||
@@ -214,7 +242,7 @@ impl DocSet for IpRangeDocSet {
|
||||
if let Some(docid) = self.loaded_docs.next() {
|
||||
docid as u32
|
||||
} else {
|
||||
if self.next_fetch_start >= self.ip_addr_fast_field.num_vals() as u32 {
|
||||
if self.next_fetch_start >= self.ip_addr_fast_field.num_docs() as u32 {
|
||||
return TERMINATED;
|
||||
}
|
||||
self.fetch_block();
|
||||
@@ -310,6 +338,12 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn ip_range_regression1_test() {
|
||||
let ops = vec![doc_from_id_1(0)];
|
||||
assert!(test_ip_range_for_docs(ops).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ip_range_regression2_test() {
|
||||
let ops = vec![
|
||||
doc_from_id_1(52),
|
||||
doc_from_id_1(63),
|
||||
@@ -321,14 +355,20 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ip_range_regression2_test() {
|
||||
let ops = vec![doc_from_id_1(0)];
|
||||
fn ip_range_regression3_test() {
|
||||
let ops = vec![doc_from_id_1(1), doc_from_id_1(2), doc_from_id_1(3)];
|
||||
assert!(test_ip_range_for_docs(ops).is_ok());
|
||||
}
|
||||
|
||||
pub fn create_index_from_docs(docs: &[Doc]) -> Index {
|
||||
let mut schema_builder = Schema::builder();
|
||||
let ip_field = schema_builder.add_ip_addr_field("ip", STORED | FAST);
|
||||
let ips_field = schema_builder.add_ip_addr_field(
|
||||
"ips",
|
||||
IpAddrOptions::default()
|
||||
.set_fast(Cardinality::MultiValues)
|
||||
.set_indexed(),
|
||||
);
|
||||
let text_field = schema_builder.add_text_field("id", STRING | STORED);
|
||||
let schema = schema_builder.build();
|
||||
let index = Index::create_in_ram(schema);
|
||||
@@ -338,6 +378,8 @@ mod tests {
|
||||
for doc in docs.iter() {
|
||||
index_writer
|
||||
.add_document(doc!(
|
||||
ips_field => doc.ip,
|
||||
ips_field => doc.ip,
|
||||
ip_field => doc.ip,
|
||||
text_field => doc.id.to_string(),
|
||||
))
|
||||
@@ -361,8 +403,8 @@ mod tests {
|
||||
.unwrap()
|
||||
};
|
||||
|
||||
let gen_query_inclusive = |from: Ipv6Addr, to: Ipv6Addr| {
|
||||
format!("ip:[{} TO {}]", &from.to_string(), &to.to_string())
|
||||
let gen_query_inclusive = |field: &str, from: Ipv6Addr, to: Ipv6Addr| {
|
||||
format!("{}:[{} TO {}]", field, &from.to_string(), &to.to_string())
|
||||
};
|
||||
|
||||
let test_sample = |sample_docs: Vec<Doc>| {
|
||||
@@ -373,7 +415,10 @@ mod tests {
|
||||
.filter(|doc| (ips[0]..=ips[1]).contains(&doc.ip))
|
||||
.count();
|
||||
|
||||
let query = gen_query_inclusive(ips[0], ips[1]);
|
||||
let query = gen_query_inclusive("ip", ips[0], ips[1]);
|
||||
assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
|
||||
|
||||
let query = gen_query_inclusive("ips", ips[0], ips[1]);
|
||||
assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
|
||||
|
||||
// Intersection search
|
||||
@@ -382,7 +427,20 @@ mod tests {
|
||||
.iter()
|
||||
.filter(|doc| (ips[0]..=ips[1]).contains(&doc.ip) && doc.id == id_filter)
|
||||
.count();
|
||||
let query = format!("{} AND id:{}", query, &id_filter);
|
||||
let query = format!(
|
||||
"{} AND id:{}",
|
||||
gen_query_inclusive("ip", ips[0], ips[1]),
|
||||
&id_filter
|
||||
);
|
||||
assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
|
||||
|
||||
// Intersection search on multivalue ip field
|
||||
let id_filter = sample_docs[0].id.to_string();
|
||||
let query = format!(
|
||||
"{} AND id:{}",
|
||||
gen_query_inclusive("ips", ips[0], ips[1]),
|
||||
&id_filter
|
||||
);
|
||||
assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
|
||||
};
|
||||
|
||||
@@ -435,6 +493,7 @@ mod bench {
|
||||
index
|
||||
}
|
||||
fn excute_query(
|
||||
field: &str,
|
||||
start_inclusive: Ipv6Addr,
|
||||
end_inclusive: Ipv6Addr,
|
||||
suffix: &str,
|
||||
@@ -442,7 +501,8 @@ mod bench {
|
||||
) -> usize {
|
||||
let gen_query_inclusive = |from: Ipv6Addr, to: Ipv6Addr| {
|
||||
format!(
|
||||
"ip:[{} TO {}] {}",
|
||||
"{}:[{} TO {}] {}",
|
||||
field,
|
||||
&from.to_string(),
|
||||
&to.to_string(),
|
||||
suffix
|
||||
@@ -469,7 +529,7 @@ mod bench {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(90 * 1000);
|
||||
|
||||
excute_query(start, end, "", &index)
|
||||
excute_query("ip", start, end, "", &index)
|
||||
});
|
||||
}
|
||||
|
||||
@@ -481,7 +541,7 @@ mod bench {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query(start, end, "", &index)
|
||||
excute_query("ip", start, end, "", &index)
|
||||
});
|
||||
}
|
||||
|
||||
@@ -493,7 +553,7 @@ mod bench {
|
||||
let start = Ipv6Addr::from_u128(10 * 1000);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query(start, end, "", &index)
|
||||
excute_query("ip", start, end, "", &index)
|
||||
});
|
||||
}
|
||||
|
||||
@@ -505,7 +565,7 @@ mod bench {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query(start, end, "AND id:few", &index)
|
||||
excute_query("ip", start, end, "AND id:few", &index)
|
||||
});
|
||||
}
|
||||
|
||||
@@ -517,7 +577,7 @@ mod bench {
|
||||
let start = Ipv6Addr::from_u128(10 * 1000);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query(start, end, "AND id:few", &index)
|
||||
excute_query("ip", start, end, "AND id:few", &index)
|
||||
});
|
||||
}
|
||||
|
||||
@@ -529,7 +589,7 @@ mod bench {
|
||||
let start = Ipv6Addr::from_u128(10 * 1000);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query(start, end, "AND id:many", &index)
|
||||
excute_query("ip", start, end, "AND id:many", &index)
|
||||
});
|
||||
}
|
||||
|
||||
@@ -541,7 +601,7 @@ mod bench {
|
||||
let start = Ipv6Addr::from_u128(10 * 1000);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query(start, end, "AND id:veryfew", &index)
|
||||
excute_query("ip", start, end, "AND id:veryfew", &index)
|
||||
});
|
||||
}
|
||||
|
||||
@@ -553,7 +613,7 @@ mod bench {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query(start, end, "AND id:many", &index)
|
||||
excute_query("ip", start, end, "AND id:many", &index)
|
||||
});
|
||||
}
|
||||
|
||||
@@ -565,7 +625,7 @@ mod bench {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(90 * 1000);
|
||||
|
||||
excute_query(start, end, "AND id:many", &index)
|
||||
excute_query("ip", start, end, "AND id:many", &index)
|
||||
});
|
||||
}
|
||||
|
||||
@@ -577,7 +637,7 @@ mod bench {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(90 * 1000);
|
||||
|
||||
excute_query(start, end, "AND id:few", &index)
|
||||
excute_query("ip", start, end, "AND id:few", &index)
|
||||
});
|
||||
}
|
||||
|
||||
@@ -589,7 +649,139 @@ mod bench {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(90 * 1000);
|
||||
|
||||
excute_query(start, end, "AND id:veryfew", &index)
|
||||
excute_query("ip", start, end, "AND id:veryfew", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_90_percent_multi(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(90 * 1000);
|
||||
|
||||
excute_query("ips", start, end, "", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_10_percent_multi(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query("ips", start, end, "", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_1_percent_multi(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(10 * 1000);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query("ips", start, end, "", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_10_percent_intersect_with_10_percent_multi(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query("ips", start, end, "AND id:few", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_1_percent_intersect_with_10_percent_multi(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(10 * 1000);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query("ips", start, end, "AND id:few", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_1_percent_intersect_with_90_percent_multi(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(10 * 1000);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query("ips", start, end, "AND id:many", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_1_percent_intersect_with_1_percent_multi(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(10 * 1000);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query("ips", start, end, "AND id:veryfew", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_10_percent_intersect_with_90_percent_multi(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query("ips", start, end, "AND id:many", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_90_percent_intersect_with_90_percent_multi(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(90 * 1000);
|
||||
|
||||
excute_query("ips", start, end, "AND id:many", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_90_percent_intersect_with_10_percent_multi(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(90 * 1000);
|
||||
|
||||
excute_query("ips", start, end, "AND id:few", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_90_percent_intersect_with_1_percent_multi(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(90 * 1000);
|
||||
|
||||
excute_query("ips", start, end, "AND id:veryfew", &index)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user