mirror of
https://github.com/quickwit-oss/tantivy.git
synced 2025-12-29 05:22:55 +00:00
Compare commits
1 Commits
trinity/yo
...
simplify
| Author | SHA1 | Date | |
|---|---|---|---|
|
fb6d5acb82 |
30
CHANGELOG.md
30
CHANGELOG.md
@@ -1,32 +1,10 @@
|
||||
Tantivy 0.19
|
||||
================================
|
||||
|
||||
- Limit fast fields to u32 (`get_val(u32)`) [#1644](https://github.com/quickwit-oss/tantivy/pull/1644) (@PSeitz)
|
||||
- Major bugfix: Fix missing fieldnorms for u64, i64, f64, bool, bytes and date [#1620](https://github.com/quickwit-oss/tantivy/pull/1620) (@PSeitz)
|
||||
- Updated [Date Field Type](https://github.com/quickwit-oss/tantivy/pull/1396)
|
||||
The `DateTime` type has been updated to hold timestamps with microseconds precision.
|
||||
`DateOptions` and `DatePrecision` have been added to configure Date fields. The precision is used to hint on fast values compression. Otherwise, seconds precision is used everywhere else (i.e terms, indexing). (@evanxg852000)
|
||||
- Add IP address field type [#1553](https://github.com/quickwit-oss/tantivy/pull/1553) (@PSeitz)
|
||||
- Add boolean field type [#1382](https://github.com/quickwit-oss/tantivy/pull/1382) (@boraarslan)
|
||||
- Remove Searcher pool and make `Searcher` cloneable. (@PSeitz)
|
||||
- Validate settings on create [#1570](https://github.com/quickwit-oss/tantivy/pull/1570 (@PSeitz)
|
||||
- Fix interpolation overflow in linear interpolation fastfield codec [#1480](https://github.com/quickwit-oss/tantivy/pull/1480 (@PSeitz @fulmicoton)
|
||||
- Detect and apply gcd on fastfield codecs [#1418](https://github.com/quickwit-oss/tantivy/pull/1418) (@PSeitz)
|
||||
- Doc store
|
||||
- use separate thread to compress block store [#1389](https://github.com/quickwit-oss/tantivy/pull/1389) [#1510](https://github.com/quickwit-oss/tantivy/pull/1510 (@PSeitz @fulmicoton)
|
||||
- Expose doc store cache size [#1403](https://github.com/quickwit-oss/tantivy/pull/1403) (@PSeitz)
|
||||
- Enable compression levels for doc store [#1378](https://github.com/quickwit-oss/tantivy/pull/1378) (@PSeitz)
|
||||
- Make block size configurable [#1374](https://github.com/quickwit-oss/tantivy/pull/1374) (@kryesh)
|
||||
- Make `tantivy::TantivyError` cloneable [#1402](https://github.com/quickwit-oss/tantivy/pull/1402) (@PSeitz)
|
||||
- Add support for phrase slop in query language [#1393](https://github.com/quickwit-oss/tantivy/pull/1393) (@saroh)
|
||||
- Aggregation
|
||||
- Add support for keyed parameter in range and histgram aggregations [#1424](https://github.com/quickwit-oss/tantivy/pull/1424) (@k-yomo)
|
||||
- Add aggregation bucket limit [#1363](https://github.com/quickwit-oss/tantivy/pull/1363) (@PSeitz)
|
||||
- Faster indexing
|
||||
- [#1610](https://github.com/quickwit-oss/tantivy/pull/1610 (@PSeitz)
|
||||
- [#1594](https://github.com/quickwit-oss/tantivy/pull/1594 (@PSeitz)
|
||||
- [#1582](https://github.com/quickwit-oss/tantivy/pull/1582 (@PSeitz)
|
||||
- [#1611](https://github.com/quickwit-oss/tantivy/pull/1611 (@PSeitz)
|
||||
`DateOptions` and `DatePrecision` have been added to configure Date fields. The precision is used to hint on fast values compression. Otherwise, seconds precision is used everywhere else (i.e terms, indexing).
|
||||
- Remove Searcher pool and make `Searcher` cloneable.
|
||||
|
||||
Tantivy 0.18
|
||||
================================
|
||||
@@ -44,10 +22,6 @@ Tantivy 0.18
|
||||
- Add terms aggregation (@PSeitz)
|
||||
- Add support for zstd compression (@kryesh)
|
||||
|
||||
Tantivy 0.18.1
|
||||
================================
|
||||
- Hotfix: positions computation. #1629 (@fmassot, @fulmicoton, @PSeitz)
|
||||
|
||||
Tantivy 0.17
|
||||
================================
|
||||
|
||||
|
||||
12
Cargo.toml
12
Cargo.toml
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "tantivy"
|
||||
version = "0.19.0-dev"
|
||||
version = "0.18.0"
|
||||
authors = ["Paul Masurel <paul.masurel@gmail.com>"]
|
||||
license = "MIT"
|
||||
categories = ["database-implementations", "data-structures"]
|
||||
@@ -14,13 +14,12 @@ edition = "2021"
|
||||
rust-version = "1.62"
|
||||
|
||||
[dependencies]
|
||||
oneshot = "0.1.5"
|
||||
oneshot = "0.1.3"
|
||||
base64 = "0.13.0"
|
||||
byteorder = "1.4.3"
|
||||
crc32fast = "1.3.2"
|
||||
once_cell = "1.10.0"
|
||||
regex = { version = "1.5.5", default-features = false, features = ["std", "unicode"] }
|
||||
aho-corasick = "0.7"
|
||||
tantivy-fst = "0.4.0"
|
||||
memmap2 = { version = "0.5.3", optional = true }
|
||||
lz4_flex = { version = "0.9.2", default-features = false, features = ["checked-decode"], optional = true }
|
||||
@@ -46,7 +45,7 @@ rust-stemmers = "1.2.0"
|
||||
downcast-rs = "1.2.0"
|
||||
bitpacking = { version = "0.8.4", default-features = false, features = ["bitpacker4x"] }
|
||||
census = "0.4.0"
|
||||
rustc-hash = "1.1.0"
|
||||
fnv = "1.0.7"
|
||||
thiserror = "1.0.30"
|
||||
htmlescape = "0.3.1"
|
||||
fail = "0.5.0"
|
||||
@@ -58,10 +57,9 @@ lru = "0.7.5"
|
||||
fastdivide = "0.4.0"
|
||||
itertools = "0.10.3"
|
||||
measure_time = "0.8.2"
|
||||
ciborium = { version = "0.2", optional = true}
|
||||
serde_cbor = { version = "0.11.2", optional = true }
|
||||
async-trait = "0.1.53"
|
||||
arc-swap = "1.5.0"
|
||||
yoke = { version = "0.6.2", features = ["derive"] }
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
winapi = "0.3.9"
|
||||
@@ -103,7 +101,7 @@ zstd-compression = ["zstd"]
|
||||
failpoints = ["fail/failpoints"]
|
||||
unstable = [] # useful for benches.
|
||||
|
||||
quickwit = ["ciborium"]
|
||||
quickwit = ["serde_cbor"]
|
||||
|
||||
[workspace]
|
||||
members = ["query-grammar", "bitpacker", "common", "fastfield_codecs", "ownedbytes"]
|
||||
|
||||
100000
benches/hdfs_with_array.json
100000
benches/hdfs_with_array.json
File diff suppressed because it is too large
Load Diff
@@ -1,159 +1,116 @@
|
||||
use criterion::{criterion_group, criterion_main, Criterion};
|
||||
use itertools::Itertools;
|
||||
use pprof::criterion::{Output, PProfProfiler};
|
||||
use serde_json::{self, Value as JsonValue};
|
||||
use tantivy::directory::RamDirectory;
|
||||
use tantivy::schema::{
|
||||
FieldValue, TextFieldIndexing, TextOptions, Value, INDEXED, STORED, STRING, TEXT,
|
||||
};
|
||||
use tantivy::{Document, Index, IndexBuilder};
|
||||
use tantivy::schema::{INDEXED, STORED, STRING, TEXT};
|
||||
use tantivy::Index;
|
||||
|
||||
const HDFS_LOGS: &str = include_str!("hdfs.json");
|
||||
const NUM_REPEATS: usize = 20;
|
||||
const NUM_REPEATS: usize = 2;
|
||||
|
||||
pub fn hdfs_index_benchmark(c: &mut Criterion) {
|
||||
let mut schema_builder = tantivy::schema::SchemaBuilder::new();
|
||||
let text_indexing_options = TextFieldIndexing::default()
|
||||
.set_tokenizer("default")
|
||||
.set_fieldnorms(false)
|
||||
.set_index_option(tantivy::schema::IndexRecordOption::WithFreqsAndPositions);
|
||||
let mut text_options = TextOptions::default().set_indexing_options(text_indexing_options);
|
||||
let text_field = schema_builder.add_text_field("body", text_options);
|
||||
let schema = schema_builder.build();
|
||||
|
||||
// prepare doc
|
||||
let mut documents_no_array = Vec::new();
|
||||
let mut documents_with_array = Vec::new();
|
||||
for doc_json in HDFS_LOGS.trim().split("\n") {
|
||||
let json_obj: serde_json::Map<String, JsonValue> = serde_json::from_str(doc_json).unwrap();
|
||||
let text = json_obj.get("body").unwrap().as_str().unwrap();
|
||||
let mut doc_no_array = Document::new();
|
||||
doc_no_array.add_text(text_field, text);
|
||||
documents_no_array.push(doc_no_array);
|
||||
let mut doc_with_array = Document::new();
|
||||
doc_with_array.add_borrowed_values(text.to_owned(), |text| {
|
||||
text.split(' ')
|
||||
.map(|text| FieldValue::new(text_field, text.into()))
|
||||
.collect()
|
||||
});
|
||||
documents_with_array.push(doc_with_array);
|
||||
}
|
||||
let schema = {
|
||||
let mut schema_builder = tantivy::schema::SchemaBuilder::new();
|
||||
schema_builder.add_u64_field("timestamp", INDEXED);
|
||||
schema_builder.add_text_field("body", TEXT);
|
||||
schema_builder.add_text_field("severity", STRING);
|
||||
schema_builder.build()
|
||||
};
|
||||
let schema_with_store = {
|
||||
let mut schema_builder = tantivy::schema::SchemaBuilder::new();
|
||||
schema_builder.add_u64_field("timestamp", INDEXED | STORED);
|
||||
schema_builder.add_text_field("body", TEXT | STORED);
|
||||
schema_builder.add_text_field("severity", STRING | STORED);
|
||||
schema_builder.build()
|
||||
};
|
||||
let dynamic_schema = {
|
||||
let mut schema_builder = tantivy::schema::SchemaBuilder::new();
|
||||
schema_builder.add_json_field("json", TEXT);
|
||||
schema_builder.build()
|
||||
};
|
||||
|
||||
let mut group = c.benchmark_group("index-hdfs");
|
||||
group.sample_size(20);
|
||||
group.bench_function("index-hdfs-no-commit", |b| {
|
||||
b.iter(|| {
|
||||
let ram_directory = RamDirectory::create();
|
||||
let mut index_writer = IndexBuilder::new()
|
||||
.schema(schema.clone())
|
||||
.single_segment_index_writer(ram_directory, 100_000_000)
|
||||
.unwrap();
|
||||
let index = Index::create_in_ram(schema.clone());
|
||||
let index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
|
||||
for _ in 0..NUM_REPEATS {
|
||||
let documents_cloned = documents_no_array.clone();
|
||||
for doc in documents_cloned {
|
||||
for doc_json in HDFS_LOGS.trim().split("\n") {
|
||||
let doc = schema.parse_document(doc_json).unwrap();
|
||||
index_writer.add_document(doc).unwrap();
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
group.bench_function("index-hdfs-with-array-no-commit", |b| {
|
||||
group.bench_function("index-hdfs-with-commit", |b| {
|
||||
b.iter(|| {
|
||||
let ram_directory = RamDirectory::create();
|
||||
let mut index_writer = IndexBuilder::new()
|
||||
.schema(schema.clone())
|
||||
.single_segment_index_writer(ram_directory, 100_000_000)
|
||||
.unwrap();
|
||||
let index = Index::create_in_ram(schema.clone());
|
||||
let mut index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
|
||||
for _ in 0..NUM_REPEATS {
|
||||
let documents_with_array_cloned = documents_with_array.clone();
|
||||
for doc in documents_with_array_cloned {
|
||||
for doc_json in HDFS_LOGS.trim().split("\n") {
|
||||
let doc = schema.parse_document(doc_json).unwrap();
|
||||
index_writer.add_document(doc).unwrap();
|
||||
}
|
||||
}
|
||||
index_writer.commit().unwrap();
|
||||
})
|
||||
});
|
||||
group.bench_function("index-hdfs-no-commit-with-docstore", |b| {
|
||||
b.iter(|| {
|
||||
let index = Index::create_in_ram(schema_with_store.clone());
|
||||
let index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
|
||||
for _ in 0..NUM_REPEATS {
|
||||
for doc_json in HDFS_LOGS.trim().split("\n") {
|
||||
let doc = schema.parse_document(doc_json).unwrap();
|
||||
index_writer.add_document(doc).unwrap();
|
||||
}
|
||||
}
|
||||
})
|
||||
});
|
||||
// group.bench_function("index-hdfs-with-commit", |b| {
|
||||
// b.iter(|| {
|
||||
// let ram_directory = RamDirectory::create();
|
||||
// let mut index_writer = IndexBuilder::new()
|
||||
// .schema(schema.clone())
|
||||
// .single_segment_index_writer(ram_directory, 100_000_000)
|
||||
// .unwrap();
|
||||
// for _ in 0..NUM_REPEATS {
|
||||
// for doc_json in HDFS_LOGS.trim().split("\n") {
|
||||
// let doc = schema.parse_document(doc_json).unwrap();
|
||||
// index_writer.add_document(doc).unwrap();
|
||||
// }
|
||||
// }
|
||||
// index_writer.commit().unwrap();
|
||||
// })
|
||||
// });
|
||||
// group.bench_function("index-hdfs-no-commit-with-docstore", |b| {
|
||||
// b.iter(|| {
|
||||
// let ram_directory = RamDirectory::create();
|
||||
// let mut index_writer = IndexBuilder::new()
|
||||
// .schema(schema.clone())
|
||||
// .single_segment_index_writer(ram_directory, 100_000_000)
|
||||
// .unwrap();
|
||||
// for _ in 0..NUM_REPEATS {
|
||||
// for doc_json in HDFS_LOGS.trim().split("\n") {
|
||||
// let doc = schema.parse_document(doc_json).unwrap();
|
||||
// index_writer.add_document(doc).unwrap();
|
||||
// }
|
||||
// }
|
||||
// })
|
||||
// });
|
||||
// group.bench_function("index-hdfs-with-commit-with-docstore", |b| {
|
||||
// b.iter(|| {
|
||||
// let ram_directory = RamDirectory::create();
|
||||
// let mut index_writer = IndexBuilder::new()
|
||||
// .schema(schema.clone())
|
||||
// .single_segment_index_writer(ram_directory, 100_000_000)
|
||||
// .unwrap();
|
||||
// for _ in 0..NUM_REPEATS {
|
||||
// for doc_json in HDFS_LOGS.trim().split("\n") {
|
||||
// let doc = schema.parse_document(doc_json).unwrap();
|
||||
// index_writer.add_document(doc).unwrap();
|
||||
// }
|
||||
// }
|
||||
// index_writer.commit().unwrap();
|
||||
// })
|
||||
// });
|
||||
// group.bench_function("index-hdfs-no-commit-json-without-docstore", |b| {
|
||||
// b.iter(|| {
|
||||
// let ram_directory = RamDirectory::create();
|
||||
// let mut index_writer = IndexBuilder::new()
|
||||
// .schema(schema.clone())
|
||||
// .single_segment_index_writer(ram_directory, 100_000_000)
|
||||
// .unwrap();
|
||||
// for _ in 0..NUM_REPEATS {
|
||||
// for doc_json in HDFS_LOGS.trim().split("\n") {
|
||||
// let json_val: serde_json::Map<String, serde_json::Value> =
|
||||
// serde_json::from_str(doc_json).unwrap();
|
||||
// let doc = tantivy::doc!(json_field=>json_val);
|
||||
// index_writer.add_document(doc).unwrap();
|
||||
// }
|
||||
// }
|
||||
// index_writer.commit().unwrap();
|
||||
// })
|
||||
// });
|
||||
// group.bench_function("index-hdfs-with-commit-json-without-docstore", |b| {
|
||||
// b.iter(|| {
|
||||
// let ram_directory = RamDirectory::create();
|
||||
// let mut index_writer = IndexBuilder::new()
|
||||
// .schema(schema.clone())
|
||||
// .single_segment_index_writer(ram_directory, 100_000_000)
|
||||
// .unwrap();
|
||||
// for _ in 0..NUM_REPEATS {
|
||||
// for doc_json in HDFS_LOGS.trim().split("\n") {
|
||||
// let json_val: serde_json::Map<String, serde_json::Value> =
|
||||
// serde_json::from_str(doc_json).unwrap();
|
||||
// let doc = tantivy::doc!(json_field=>json_val);
|
||||
// index_writer.add_document(doc).unwrap();
|
||||
// }
|
||||
// }
|
||||
// index_writer.commit().unwrap();
|
||||
// })
|
||||
//});
|
||||
group.bench_function("index-hdfs-with-commit-with-docstore", |b| {
|
||||
b.iter(|| {
|
||||
let index = Index::create_in_ram(schema_with_store.clone());
|
||||
let mut index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
|
||||
for _ in 0..NUM_REPEATS {
|
||||
for doc_json in HDFS_LOGS.trim().split("\n") {
|
||||
let doc = schema.parse_document(doc_json).unwrap();
|
||||
index_writer.add_document(doc).unwrap();
|
||||
}
|
||||
}
|
||||
index_writer.commit().unwrap();
|
||||
})
|
||||
});
|
||||
group.bench_function("index-hdfs-no-commit-json-without-docstore", |b| {
|
||||
b.iter(|| {
|
||||
let index = Index::create_in_ram(dynamic_schema.clone());
|
||||
let json_field = dynamic_schema.get_field("json").unwrap();
|
||||
let mut index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
|
||||
for _ in 0..NUM_REPEATS {
|
||||
for doc_json in HDFS_LOGS.trim().split("\n") {
|
||||
let json_val: serde_json::Map<String, serde_json::Value> =
|
||||
serde_json::from_str(doc_json).unwrap();
|
||||
let doc = tantivy::doc!(json_field=>json_val);
|
||||
index_writer.add_document(doc).unwrap();
|
||||
}
|
||||
}
|
||||
index_writer.commit().unwrap();
|
||||
})
|
||||
});
|
||||
group.bench_function("index-hdfs-with-commit-json-without-docstore", |b| {
|
||||
b.iter(|| {
|
||||
let index = Index::create_in_ram(dynamic_schema.clone());
|
||||
let json_field = dynamic_schema.get_field("json").unwrap();
|
||||
let mut index_writer = index.writer_with_num_threads(1, 100_000_000).unwrap();
|
||||
for _ in 0..NUM_REPEATS {
|
||||
for doc_json in HDFS_LOGS.trim().split("\n") {
|
||||
let json_val: serde_json::Map<String, serde_json::Value> =
|
||||
serde_json::from_str(doc_json).unwrap();
|
||||
let doc = tantivy::doc!(json_field=>json_val);
|
||||
index_writer.add_document(doc).unwrap();
|
||||
}
|
||||
}
|
||||
index_writer.commit().unwrap();
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
criterion_group! {
|
||||
|
||||
@@ -87,15 +87,15 @@ impl BitUnpacker {
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn get(&self, idx: u32, data: &[u8]) -> u64 {
|
||||
pub fn get(&self, idx: u64, data: &[u8]) -> u64 {
|
||||
if self.num_bits == 0 {
|
||||
return 0u64;
|
||||
}
|
||||
let addr_in_bits = idx * self.num_bits as u32;
|
||||
let addr_in_bits = idx * self.num_bits;
|
||||
let addr = addr_in_bits >> 3;
|
||||
let bit_shift = addr_in_bits & 7;
|
||||
debug_assert!(
|
||||
addr + 8 <= data.len() as u32,
|
||||
addr + 8 <= data.len() as u64,
|
||||
"The fast field field should have been padded with 7 bytes."
|
||||
);
|
||||
let bytes: [u8; 8] = (&data[(addr as usize)..(addr as usize) + 8])
|
||||
@@ -130,7 +130,7 @@ mod test {
|
||||
fn test_bitpacker_util(len: usize, num_bits: u8) {
|
||||
let (bitunpacker, vals, data) = create_fastfield_bitpacker(len, num_bits);
|
||||
for (i, val) in vals.iter().enumerate() {
|
||||
assert_eq!(bitunpacker.get(i as u32, &data), *val);
|
||||
assert_eq!(bitunpacker.get(i as u64, &data), *val);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -130,7 +130,7 @@ impl BlockedBitpacker {
|
||||
let pos_in_block = idx % BLOCK_SIZE as usize;
|
||||
if let Some(metadata) = self.offset_and_bits.get(metadata_pos) {
|
||||
let unpacked = BitUnpacker::new(metadata.num_bits()).get(
|
||||
pos_in_block as u32,
|
||||
pos_in_block as u64,
|
||||
&self.compressed_blocks[metadata.offset() as usize..],
|
||||
);
|
||||
unpacked + metadata.base_value()
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
use std::borrow::Cow;
|
||||
use std::io::{Read, Write};
|
||||
use std::{fmt, io};
|
||||
|
||||
@@ -108,19 +107,6 @@ impl FixedSize for u64 {
|
||||
const SIZE_IN_BYTES: usize = 8;
|
||||
}
|
||||
|
||||
impl BinarySerializable for u128 {
|
||||
fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
|
||||
writer.write_u128::<Endianness>(*self)
|
||||
}
|
||||
fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {
|
||||
reader.read_u128::<Endianness>()
|
||||
}
|
||||
}
|
||||
|
||||
impl FixedSize for u128 {
|
||||
const SIZE_IN_BYTES: usize = 16;
|
||||
}
|
||||
|
||||
impl BinarySerializable for f32 {
|
||||
fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
|
||||
writer.write_f32::<Endianness>(*self)
|
||||
@@ -211,23 +197,6 @@ impl BinarySerializable for String {
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> BinarySerializable for Cow<'a, str> {
|
||||
fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
|
||||
let data: &[u8] = self.as_bytes();
|
||||
VInt(data.len() as u64).serialize(writer)?;
|
||||
writer.write_all(data)
|
||||
}
|
||||
|
||||
fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {
|
||||
let string_length = VInt::deserialize(reader)?.val() as usize;
|
||||
let mut result = String::with_capacity(string_length);
|
||||
reader
|
||||
.take(string_length as u64)
|
||||
.read_to_string(&mut result)?;
|
||||
Ok(Cow::Owned(result))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub mod test {
|
||||
|
||||
|
||||
@@ -105,7 +105,7 @@ impl SegmentCollector for StatsSegmentCollector {
|
||||
type Fruit = Option<Stats>;
|
||||
|
||||
fn collect(&mut self, doc: u32, _score: Score) {
|
||||
let value = self.fast_field_reader.get_val(doc) as f64;
|
||||
let value = self.fast_field_reader.get_val(doc as u64) as f64;
|
||||
self.stats.count += 1;
|
||||
self.stats.sum += value;
|
||||
self.stats.squared_sum += value * value;
|
||||
|
||||
@@ -51,7 +51,7 @@ impl Warmer for DynamicPriceColumn {
|
||||
let product_id_reader = segment.fast_fields().u64(self.field)?;
|
||||
let product_ids: Vec<ProductId> = segment
|
||||
.doc_ids_alive()
|
||||
.map(|doc| product_id_reader.get_val(doc))
|
||||
.map(|doc| product_id_reader.get_val(doc as u64))
|
||||
.collect();
|
||||
let mut prices_it = self.price_fetcher.fetch_prices(&product_ids).into_iter();
|
||||
let mut price_vals: Vec<Price> = Vec::new();
|
||||
|
||||
@@ -65,7 +65,7 @@ mod tests {
|
||||
b.iter(|| {
|
||||
let mut a = 0u64;
|
||||
for _ in 0..n {
|
||||
a = column.get_val(a as u32);
|
||||
a = column.get_val(a as u64);
|
||||
}
|
||||
a
|
||||
});
|
||||
@@ -100,10 +100,9 @@ mod tests {
|
||||
|
||||
fn get_u128_column_from_data(data: &[u128]) -> Arc<dyn Column<u128>> {
|
||||
let mut out = vec![];
|
||||
let iter_gen = || data.iter().cloned();
|
||||
serialize_u128(iter_gen, data.len() as u32, &mut out).unwrap();
|
||||
serialize_u128(VecColumn::from(&data), &mut out).unwrap();
|
||||
let out = OwnedBytes::new(out);
|
||||
open_u128::<u128>(out).unwrap()
|
||||
open_u128(out).unwrap()
|
||||
}
|
||||
|
||||
#[bench]
|
||||
@@ -111,15 +110,7 @@ mod tests {
|
||||
let (major_item, _minor_item, data) = get_data_50percent_item();
|
||||
let column = get_u128_column_from_data(&data);
|
||||
|
||||
b.iter(|| {
|
||||
let mut positions = Vec::new();
|
||||
column.get_positions_for_value_range(
|
||||
major_item..=major_item,
|
||||
0..data.len() as u32,
|
||||
&mut positions,
|
||||
);
|
||||
positions
|
||||
});
|
||||
b.iter(|| column.get_between_vals(major_item..=major_item));
|
||||
}
|
||||
|
||||
#[bench]
|
||||
@@ -127,15 +118,7 @@ mod tests {
|
||||
let (_major_item, minor_item, data) = get_data_50percent_item();
|
||||
let column = get_u128_column_from_data(&data);
|
||||
|
||||
b.iter(|| {
|
||||
let mut positions = Vec::new();
|
||||
column.get_positions_for_value_range(
|
||||
minor_item..=minor_item,
|
||||
0..data.len() as u32,
|
||||
&mut positions,
|
||||
);
|
||||
positions
|
||||
});
|
||||
b.iter(|| column.get_between_vals(minor_item..=minor_item));
|
||||
}
|
||||
|
||||
#[bench]
|
||||
@@ -143,15 +126,7 @@ mod tests {
|
||||
let (_major_item, _minor_item, data) = get_data_50percent_item();
|
||||
let column = get_u128_column_from_data(&data);
|
||||
|
||||
b.iter(|| {
|
||||
let mut positions = Vec::new();
|
||||
column.get_positions_for_value_range(
|
||||
0..=u128::MAX,
|
||||
0..data.len() as u32,
|
||||
&mut positions,
|
||||
);
|
||||
positions
|
||||
});
|
||||
b.iter(|| column.get_between_vals(0..=u128::MAX));
|
||||
}
|
||||
|
||||
#[bench]
|
||||
@@ -161,7 +136,7 @@ mod tests {
|
||||
b.iter(|| {
|
||||
let mut a = 0u128;
|
||||
for i in 0u64..column.num_vals() as u64 {
|
||||
a += column.get_val(i as u32);
|
||||
a += column.get_val(i);
|
||||
}
|
||||
a
|
||||
});
|
||||
@@ -175,7 +150,7 @@ mod tests {
|
||||
let n = column.num_vals();
|
||||
let mut a = 0u128;
|
||||
for i in (0..n / 5).map(|val| val * 5) {
|
||||
a += column.get_val(i);
|
||||
a += column.get_val(i as u64);
|
||||
}
|
||||
a
|
||||
});
|
||||
@@ -200,9 +175,9 @@ mod tests {
|
||||
let n = permutation.len();
|
||||
let column: Arc<dyn Column<u64>> = serialize_and_load(&permutation);
|
||||
b.iter(|| {
|
||||
let mut a = 0;
|
||||
let mut a = 0u64;
|
||||
for i in (0..n / 7).map(|val| val * 7) {
|
||||
a += column.get_val(i as u32);
|
||||
a += column.get_val(i as u64);
|
||||
}
|
||||
a
|
||||
});
|
||||
@@ -215,7 +190,7 @@ mod tests {
|
||||
let column: Arc<dyn Column<u64>> = serialize_and_load(&permutation);
|
||||
b.iter(|| {
|
||||
let mut a = 0u64;
|
||||
for i in 0u32..n as u32 {
|
||||
for i in 0u64..n as u64 {
|
||||
a += column.get_val(i);
|
||||
}
|
||||
a
|
||||
@@ -229,8 +204,8 @@ mod tests {
|
||||
let column: Arc<dyn Column<u64>> = serialize_and_load(&permutation);
|
||||
b.iter(|| {
|
||||
let mut a = 0u64;
|
||||
for i in 0..n {
|
||||
a += column.get_val(i as u32);
|
||||
for i in 0..n as u64 {
|
||||
a += column.get_val(i);
|
||||
}
|
||||
a
|
||||
});
|
||||
|
||||
@@ -17,7 +17,7 @@ pub struct BitpackedReader {
|
||||
|
||||
impl Column for BitpackedReader {
|
||||
#[inline]
|
||||
fn get_val(&self, doc: u32) -> u64 {
|
||||
fn get_val(&self, doc: u64) -> u64 {
|
||||
self.bit_unpacker.get(doc, &self.data)
|
||||
}
|
||||
#[inline]
|
||||
@@ -30,7 +30,7 @@ impl Column for BitpackedReader {
|
||||
self.normalized_header.max_value
|
||||
}
|
||||
#[inline]
|
||||
fn num_vals(&self) -> u32 {
|
||||
fn num_vals(&self) -> u64 {
|
||||
self.normalized_header.num_vals
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@ impl BinarySerializable for Block {
|
||||
}
|
||||
}
|
||||
|
||||
fn compute_num_blocks(num_vals: u32) -> usize {
|
||||
fn compute_num_blocks(num_vals: u64) -> usize {
|
||||
(num_vals as usize + CHUNK_SIZE - 1) / CHUNK_SIZE
|
||||
}
|
||||
|
||||
@@ -72,13 +72,13 @@ impl FastFieldCodec for BlockwiseLinearCodec {
|
||||
|
||||
// Estimate first_chunk and extrapolate
|
||||
fn estimate(column: &dyn crate::Column) -> Option<f32> {
|
||||
if column.num_vals() < 10 * CHUNK_SIZE as u32 {
|
||||
if column.num_vals() < 10 * CHUNK_SIZE as u64 {
|
||||
return None;
|
||||
}
|
||||
let mut first_chunk: Vec<u64> = column.iter().take(CHUNK_SIZE as usize).collect();
|
||||
let line = Line::train(&VecColumn::from(&first_chunk));
|
||||
for (i, buffer_val) in first_chunk.iter_mut().enumerate() {
|
||||
let interpolated_val = line.eval(i as u32);
|
||||
let interpolated_val = line.eval(i as u64);
|
||||
*buffer_val = buffer_val.wrapping_sub(interpolated_val);
|
||||
}
|
||||
let estimated_bit_width = first_chunk
|
||||
@@ -95,7 +95,7 @@ impl FastFieldCodec for BlockwiseLinearCodec {
|
||||
};
|
||||
let num_bits = estimated_bit_width as u64 * column.num_vals() as u64
|
||||
// function metadata per block
|
||||
+ metadata_per_block as u64 * (column.num_vals() as u64 / CHUNK_SIZE as u64);
|
||||
+ metadata_per_block as u64 * (column.num_vals() / CHUNK_SIZE as u64);
|
||||
let num_bits_uncompressed = 64 * column.num_vals();
|
||||
Some(num_bits as f32 / num_bits_uncompressed as f32)
|
||||
}
|
||||
@@ -121,7 +121,7 @@ impl FastFieldCodec for BlockwiseLinearCodec {
|
||||
assert!(!buffer.is_empty());
|
||||
|
||||
for (i, buffer_val) in buffer.iter_mut().enumerate() {
|
||||
let interpolated_val = line.eval(i as u32);
|
||||
let interpolated_val = line.eval(i as u64);
|
||||
*buffer_val = buffer_val.wrapping_sub(interpolated_val);
|
||||
}
|
||||
let bit_width = buffer.iter().copied().map(compute_num_bits).max().unwrap();
|
||||
@@ -161,9 +161,9 @@ pub struct BlockwiseLinearReader {
|
||||
|
||||
impl Column for BlockwiseLinearReader {
|
||||
#[inline(always)]
|
||||
fn get_val(&self, idx: u32) -> u64 {
|
||||
let block_id = (idx / CHUNK_SIZE as u32) as usize;
|
||||
let idx_within_block = idx % (CHUNK_SIZE as u32);
|
||||
fn get_val(&self, idx: u64) -> u64 {
|
||||
let block_id = (idx / CHUNK_SIZE as u64) as usize;
|
||||
let idx_within_block = idx % (CHUNK_SIZE as u64);
|
||||
let block = &self.blocks[block_id];
|
||||
let interpoled_val: u64 = block.line.eval(idx_within_block);
|
||||
let block_bytes = &self.data[block.data_start_offset..];
|
||||
@@ -180,7 +180,7 @@ impl Column for BlockwiseLinearReader {
|
||||
self.normalized_header.max_value
|
||||
}
|
||||
|
||||
fn num_vals(&self) -> u32 {
|
||||
fn num_vals(&self) -> u64 {
|
||||
self.normalized_header.num_vals
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
use std::marker::PhantomData;
|
||||
use std::ops::{Range, RangeInclusive};
|
||||
use std::ops::RangeInclusive;
|
||||
|
||||
use tantivy_bitpacker::minmax;
|
||||
|
||||
use crate::monotonic_mapping::StrictlyMonotonicFn;
|
||||
|
||||
/// `Column` provides columnar access on a field.
|
||||
pub trait Column<T: PartialOrd = u64>: Send + Sync {
|
||||
/// Return the value associated with the given idx.
|
||||
///
|
||||
@@ -14,7 +11,7 @@ pub trait Column<T: PartialOrd = u64>: Send + Sync {
|
||||
/// # Panics
|
||||
///
|
||||
/// May panic if `idx` is greater than the column length.
|
||||
fn get_val(&self, idx: u32) -> T;
|
||||
fn get_val(&self, idx: u64) -> T;
|
||||
|
||||
/// Fills an output buffer with the fast field values
|
||||
/// associated with the `DocId` going from
|
||||
@@ -27,28 +24,21 @@ pub trait Column<T: PartialOrd = u64>: Send + Sync {
|
||||
#[inline]
|
||||
fn get_range(&self, start: u64, output: &mut [T]) {
|
||||
for (out, idx) in output.iter_mut().zip(start..) {
|
||||
*out = self.get_val(idx as u32);
|
||||
*out = self.get_val(idx);
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the positions of values which are in the provided value range.
|
||||
///
|
||||
/// Note that position == docid for single value fast fields
|
||||
/// Return the positions of values which are in the provided range.
|
||||
#[inline]
|
||||
fn get_positions_for_value_range(
|
||||
&self,
|
||||
value_range: RangeInclusive<T>,
|
||||
doc_id_range: Range<u32>,
|
||||
positions: &mut Vec<u32>,
|
||||
) {
|
||||
let doc_id_range = doc_id_range.start..doc_id_range.end.min(self.num_vals());
|
||||
|
||||
for idx in doc_id_range.start..doc_id_range.end {
|
||||
fn get_between_vals(&self, range: RangeInclusive<T>) -> Vec<u64> {
|
||||
let mut vals = Vec::new();
|
||||
for idx in 0..self.num_vals() {
|
||||
let val = self.get_val(idx);
|
||||
if value_range.contains(&val) {
|
||||
positions.push(idx);
|
||||
if range.contains(&val) {
|
||||
vals.push(idx);
|
||||
}
|
||||
}
|
||||
vals
|
||||
}
|
||||
|
||||
/// Returns the minimum value for this fast field.
|
||||
@@ -67,8 +57,7 @@ pub trait Column<T: PartialOrd = u64>: Send + Sync {
|
||||
/// `.max_value()`.
|
||||
fn max_value(&self) -> T;
|
||||
|
||||
/// The number of values in the column.
|
||||
fn num_vals(&self) -> u32;
|
||||
fn num_vals(&self) -> u64;
|
||||
|
||||
/// Returns a iterator over the data
|
||||
fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = T> + 'a> {
|
||||
@@ -76,7 +65,6 @@ pub trait Column<T: PartialOrd = u64>: Send + Sync {
|
||||
}
|
||||
}
|
||||
|
||||
/// VecColumn provides `Column` over a slice.
|
||||
pub struct VecColumn<'a, T = u64> {
|
||||
values: &'a [T],
|
||||
min_value: T,
|
||||
@@ -84,7 +72,7 @@ pub struct VecColumn<'a, T = u64> {
|
||||
}
|
||||
|
||||
impl<'a, C: Column<T>, T: Copy + PartialOrd> Column<T> for &'a C {
|
||||
fn get_val(&self, idx: u32) -> T {
|
||||
fn get_val(&self, idx: u64) -> T {
|
||||
(*self).get_val(idx)
|
||||
}
|
||||
|
||||
@@ -96,7 +84,7 @@ impl<'a, C: Column<T>, T: Copy + PartialOrd> Column<T> for &'a C {
|
||||
(*self).max_value()
|
||||
}
|
||||
|
||||
fn num_vals(&self) -> u32 {
|
||||
fn num_vals(&self) -> u64 {
|
||||
(*self).num_vals()
|
||||
}
|
||||
|
||||
@@ -110,7 +98,7 @@ impl<'a, C: Column<T>, T: Copy + PartialOrd> Column<T> for &'a C {
|
||||
}
|
||||
|
||||
impl<'a, T: Copy + PartialOrd + Send + Sync> Column<T> for VecColumn<'a, T> {
|
||||
fn get_val(&self, position: u32) -> T {
|
||||
fn get_val(&self, position: u64) -> T {
|
||||
self.values[position as usize]
|
||||
}
|
||||
|
||||
@@ -126,8 +114,8 @@ impl<'a, T: Copy + PartialOrd + Send + Sync> Column<T> for VecColumn<'a, T> {
|
||||
self.max_value
|
||||
}
|
||||
|
||||
fn num_vals(&self) -> u32 {
|
||||
self.values.len() as u32
|
||||
fn num_vals(&self) -> u64 {
|
||||
self.values.len() as u64
|
||||
}
|
||||
|
||||
fn get_range(&self, start: u64, output: &mut [T]) {
|
||||
@@ -155,30 +143,16 @@ struct MonotonicMappingColumn<C, T, Input> {
|
||||
_phantom: PhantomData<Input>,
|
||||
}
|
||||
|
||||
/// Creates a view of a column transformed by a strictly monotonic mapping. See
|
||||
/// [`StrictlyMonotonicFn`].
|
||||
///
|
||||
/// E.g. apply a gcd monotonic_mapping([100, 200, 300]) == [1, 2, 3]
|
||||
/// monotonic_mapping.mapping() is expected to be injective, and we should always have
|
||||
/// monotonic_mapping.inverse(monotonic_mapping.mapping(el)) == el
|
||||
///
|
||||
/// The inverse of the mapping is required for:
|
||||
/// `fn get_positions_for_value_range(&self, range: RangeInclusive<T>) -> Vec<u64> `
|
||||
/// The user provides the original value range and we need to monotonic map them in the same way the
|
||||
/// serialization does before calling the underlying column.
|
||||
///
|
||||
/// Note that when opening a codec, the monotonic_mapping should be the inverse of the mapping
|
||||
/// during serialization. And therefore the monotonic_mapping_inv when opening is the same as
|
||||
/// monotonic_mapping during serialization.
|
||||
pub fn monotonic_map_column<C, T, Input, Output>(
|
||||
/// Creates a view of a column transformed by a monotonic mapping.
|
||||
pub fn monotonic_map_column<C, T, Input: PartialOrd, Output: PartialOrd>(
|
||||
from_column: C,
|
||||
monotonic_mapping: T,
|
||||
) -> impl Column<Output>
|
||||
where
|
||||
C: Column<Input>,
|
||||
T: StrictlyMonotonicFn<Input, Output> + Send + Sync,
|
||||
Input: PartialOrd + Send + Sync + Clone,
|
||||
Output: PartialOrd + Send + Sync + Clone,
|
||||
T: Fn(Input) -> Output + Send + Sync,
|
||||
Input: Send + Sync,
|
||||
Output: Send + Sync,
|
||||
{
|
||||
MonotonicMappingColumn {
|
||||
from_column,
|
||||
@@ -187,53 +161,36 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<C, T, Input, Output> Column<Output> for MonotonicMappingColumn<C, T, Input>
|
||||
impl<C, T, Input: PartialOrd, Output: PartialOrd> Column<Output>
|
||||
for MonotonicMappingColumn<C, T, Input>
|
||||
where
|
||||
C: Column<Input>,
|
||||
T: StrictlyMonotonicFn<Input, Output> + Send + Sync,
|
||||
Input: PartialOrd + Send + Sync + Clone,
|
||||
Output: PartialOrd + Send + Sync + Clone,
|
||||
T: Fn(Input) -> Output + Send + Sync,
|
||||
Input: Send + Sync,
|
||||
Output: Send + Sync,
|
||||
{
|
||||
#[inline]
|
||||
fn get_val(&self, idx: u32) -> Output {
|
||||
fn get_val(&self, idx: u64) -> Output {
|
||||
let from_val = self.from_column.get_val(idx);
|
||||
self.monotonic_mapping.mapping(from_val)
|
||||
(self.monotonic_mapping)(from_val)
|
||||
}
|
||||
|
||||
fn min_value(&self) -> Output {
|
||||
let from_min_value = self.from_column.min_value();
|
||||
self.monotonic_mapping.mapping(from_min_value)
|
||||
(self.monotonic_mapping)(from_min_value)
|
||||
}
|
||||
|
||||
fn max_value(&self) -> Output {
|
||||
let from_max_value = self.from_column.max_value();
|
||||
self.monotonic_mapping.mapping(from_max_value)
|
||||
(self.monotonic_mapping)(from_max_value)
|
||||
}
|
||||
|
||||
fn num_vals(&self) -> u32 {
|
||||
fn num_vals(&self) -> u64 {
|
||||
self.from_column.num_vals()
|
||||
}
|
||||
|
||||
fn iter(&self) -> Box<dyn Iterator<Item = Output> + '_> {
|
||||
Box::new(
|
||||
self.from_column
|
||||
.iter()
|
||||
.map(|el| self.monotonic_mapping.mapping(el)),
|
||||
)
|
||||
}
|
||||
|
||||
fn get_positions_for_value_range(
|
||||
&self,
|
||||
range: RangeInclusive<Output>,
|
||||
doc_id_range: Range<u32>,
|
||||
positions: &mut Vec<u32>,
|
||||
) {
|
||||
self.from_column.get_positions_for_value_range(
|
||||
self.monotonic_mapping.inverse(range.start().clone())
|
||||
..=self.monotonic_mapping.inverse(range.end().clone()),
|
||||
doc_id_range,
|
||||
positions,
|
||||
)
|
||||
Box::new(self.from_column.iter().map(&self.monotonic_mapping))
|
||||
}
|
||||
|
||||
// We voluntarily do not implement get_range as it yields a regression,
|
||||
@@ -255,7 +212,7 @@ where
|
||||
T: Iterator + Clone + ExactSizeIterator + Send + Sync,
|
||||
T::Item: PartialOrd,
|
||||
{
|
||||
fn get_val(&self, idx: u32) -> T::Item {
|
||||
fn get_val(&self, idx: u64) -> T::Item {
|
||||
self.0.clone().nth(idx as usize).unwrap()
|
||||
}
|
||||
|
||||
@@ -267,8 +224,8 @@ where
|
||||
self.0.clone().last().unwrap()
|
||||
}
|
||||
|
||||
fn num_vals(&self) -> u32 {
|
||||
self.0.len() as u32
|
||||
fn num_vals(&self) -> u64 {
|
||||
self.0.len() as u64
|
||||
}
|
||||
|
||||
fn iter(&self) -> Box<dyn Iterator<Item = T::Item> + '_> {
|
||||
@@ -279,22 +236,19 @@ where
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::monotonic_mapping::{
|
||||
StrictlyMonotonicMappingInverter, StrictlyMonotonicMappingToInternalBaseval,
|
||||
StrictlyMonotonicMappingToInternalGCDBaseval,
|
||||
};
|
||||
use crate::MonotonicallyMappableToU64;
|
||||
|
||||
#[test]
|
||||
fn test_monotonic_mapping() {
|
||||
let vals = &[3u64, 5u64][..];
|
||||
let vals = &[1u64, 3u64][..];
|
||||
let col = VecColumn::from(vals);
|
||||
let mapped = monotonic_map_column(col, StrictlyMonotonicMappingToInternalBaseval::new(2));
|
||||
assert_eq!(mapped.min_value(), 1u64);
|
||||
assert_eq!(mapped.max_value(), 3u64);
|
||||
let mapped = monotonic_map_column(col, |el| el + 4);
|
||||
assert_eq!(mapped.min_value(), 5u64);
|
||||
assert_eq!(mapped.max_value(), 7u64);
|
||||
assert_eq!(mapped.num_vals(), 2);
|
||||
assert_eq!(mapped.num_vals(), 2);
|
||||
assert_eq!(mapped.get_val(0), 1);
|
||||
assert_eq!(mapped.get_val(1), 3);
|
||||
assert_eq!(mapped.get_val(0), 5);
|
||||
assert_eq!(mapped.get_val(1), 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -306,15 +260,10 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_monotonic_mapping_iter() {
|
||||
let vals: Vec<u64> = (10..110u64).map(|el| el * 10).collect();
|
||||
let vals: Vec<u64> = (-1..99).map(i64::to_u64).collect();
|
||||
let col = VecColumn::from(&vals);
|
||||
let mapped = monotonic_map_column(
|
||||
col,
|
||||
StrictlyMonotonicMappingInverter::from(
|
||||
StrictlyMonotonicMappingToInternalGCDBaseval::new(10, 100),
|
||||
),
|
||||
);
|
||||
let val_i64s: Vec<u64> = mapped.iter().collect();
|
||||
let mapped = monotonic_map_column(col, |el| i64::from_u64(el) * 10i64);
|
||||
let val_i64s: Vec<i64> = mapped.iter().collect();
|
||||
for i in 0..100 {
|
||||
assert_eq!(val_i64s[i as usize], mapped.get_val(i));
|
||||
}
|
||||
@@ -322,26 +271,20 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_monotonic_mapping_get_range() {
|
||||
let vals: Vec<u64> = (0..100u64).map(|el| el * 10).collect();
|
||||
let vals: Vec<u64> = (-1..99).map(i64::to_u64).collect();
|
||||
let col = VecColumn::from(&vals);
|
||||
let mapped = monotonic_map_column(
|
||||
col,
|
||||
StrictlyMonotonicMappingInverter::from(
|
||||
StrictlyMonotonicMappingToInternalGCDBaseval::new(10, 0),
|
||||
),
|
||||
);
|
||||
|
||||
assert_eq!(mapped.min_value(), 0u64);
|
||||
assert_eq!(mapped.max_value(), 9900u64);
|
||||
let mapped = monotonic_map_column(col, |el| i64::from_u64(el) * 10i64);
|
||||
assert_eq!(mapped.min_value(), -10i64);
|
||||
assert_eq!(mapped.max_value(), 980i64);
|
||||
assert_eq!(mapped.num_vals(), 100);
|
||||
let val_u64s: Vec<u64> = mapped.iter().collect();
|
||||
assert_eq!(val_u64s.len(), 100);
|
||||
let val_i64s: Vec<i64> = mapped.iter().collect();
|
||||
assert_eq!(val_i64s.len(), 100);
|
||||
for i in 0..100 {
|
||||
assert_eq!(val_u64s[i as usize], mapped.get_val(i));
|
||||
assert_eq!(val_u64s[i as usize], vals[i as usize] * 10);
|
||||
assert_eq!(val_i64s[i as usize], mapped.get_val(i));
|
||||
assert_eq!(val_i64s[i as usize], i64::from_u64(vals[i as usize]) * 10);
|
||||
}
|
||||
let mut buf = [0u64; 20];
|
||||
let mut buf = [0i64; 20];
|
||||
mapped.get_range(7, &mut buf[..]);
|
||||
assert_eq!(&val_u64s[7..][..20], &buf);
|
||||
assert_eq!(&val_i64s[7..][..20], &buf);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,7 +57,7 @@ fn num_bits(val: u128) -> u8 {
|
||||
/// metadata.
|
||||
pub fn get_compact_space(
|
||||
values_deduped_sorted: &BTreeSet<u128>,
|
||||
total_num_values: u32,
|
||||
total_num_values: u64,
|
||||
cost_per_blank: usize,
|
||||
) -> CompactSpace {
|
||||
let mut compact_space_builder = CompactSpaceBuilder::new();
|
||||
|
||||
@@ -14,7 +14,7 @@ use std::{
|
||||
cmp::Ordering,
|
||||
collections::BTreeSet,
|
||||
io::{self, Write},
|
||||
ops::{Range, RangeInclusive},
|
||||
ops::RangeInclusive,
|
||||
};
|
||||
|
||||
use common::{BinarySerializable, CountingWriter, VInt, VIntU128};
|
||||
@@ -165,16 +165,16 @@ pub struct IPCodecParams {
|
||||
bit_unpacker: BitUnpacker,
|
||||
min_value: u128,
|
||||
max_value: u128,
|
||||
num_vals: u32,
|
||||
num_vals: u64,
|
||||
num_bits: u8,
|
||||
}
|
||||
|
||||
impl CompactSpaceCompressor {
|
||||
/// Taking the vals as Vec may cost a lot of memory. It is used to sort the vals.
|
||||
pub fn train_from(iter: impl Iterator<Item = u128>, num_vals: u32) -> Self {
|
||||
pub fn train_from(column: &impl Column<u128>) -> Self {
|
||||
let mut values_sorted = BTreeSet::new();
|
||||
values_sorted.extend(iter);
|
||||
let total_num_values = num_vals;
|
||||
values_sorted.extend(column.iter());
|
||||
let total_num_values = column.num_vals();
|
||||
|
||||
let compact_space =
|
||||
get_compact_space(&values_sorted, total_num_values, COST_PER_BLANK_IN_BITS);
|
||||
@@ -200,7 +200,7 @@ impl CompactSpaceCompressor {
|
||||
bit_unpacker: BitUnpacker::new(num_bits),
|
||||
min_value,
|
||||
max_value,
|
||||
num_vals: total_num_values,
|
||||
num_vals: total_num_values as u64,
|
||||
num_bits,
|
||||
},
|
||||
}
|
||||
@@ -267,7 +267,7 @@ impl BinarySerializable for IPCodecParams {
|
||||
let _header_flags = u64::deserialize(reader)?;
|
||||
let min_value = VIntU128::deserialize(reader)?.0;
|
||||
let max_value = VIntU128::deserialize(reader)?.0;
|
||||
let num_vals = VIntU128::deserialize(reader)?.0 as u32;
|
||||
let num_vals = VIntU128::deserialize(reader)?.0 as u64;
|
||||
let num_bits = u8::deserialize(reader)?;
|
||||
let compact_space = CompactSpace::deserialize(reader)?;
|
||||
|
||||
@@ -284,7 +284,7 @@ impl BinarySerializable for IPCodecParams {
|
||||
|
||||
impl Column<u128> for CompactSpaceDecompressor {
|
||||
#[inline]
|
||||
fn get_val(&self, doc: u32) -> u128 {
|
||||
fn get_val(&self, doc: u64) -> u128 {
|
||||
self.get(doc)
|
||||
}
|
||||
|
||||
@@ -296,7 +296,7 @@ impl Column<u128> for CompactSpaceDecompressor {
|
||||
self.max_value()
|
||||
}
|
||||
|
||||
fn num_vals(&self) -> u32 {
|
||||
fn num_vals(&self) -> u64 {
|
||||
self.params.num_vals
|
||||
}
|
||||
|
||||
@@ -304,15 +304,8 @@ impl Column<u128> for CompactSpaceDecompressor {
|
||||
fn iter(&self) -> Box<dyn Iterator<Item = u128> + '_> {
|
||||
Box::new(self.iter())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn get_positions_for_value_range(
|
||||
&self,
|
||||
value_range: RangeInclusive<u128>,
|
||||
doc_id_range: Range<u32>,
|
||||
positions: &mut Vec<u32>,
|
||||
) {
|
||||
self.get_positions_for_value_range(value_range, doc_id_range, positions)
|
||||
fn get_between_vals(&self, range: RangeInclusive<u128>) -> Vec<u64> {
|
||||
self.get_between_vals(range)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -347,19 +340,12 @@ impl CompactSpaceDecompressor {
|
||||
/// Comparing on compact space: Real dataset 1.08 GElements/s
|
||||
///
|
||||
/// Comparing on original space: Real dataset .06 GElements/s (not completely optimized)
|
||||
#[inline]
|
||||
pub fn get_positions_for_value_range(
|
||||
&self,
|
||||
value_range: RangeInclusive<u128>,
|
||||
doc_id_range: Range<u32>,
|
||||
positions: &mut Vec<u32>,
|
||||
) {
|
||||
if value_range.start() > value_range.end() {
|
||||
return;
|
||||
pub fn get_between_vals(&self, range: RangeInclusive<u128>) -> Vec<u64> {
|
||||
if range.start() > range.end() {
|
||||
return Vec::new();
|
||||
}
|
||||
let doc_id_range = doc_id_range.start..doc_id_range.end.min(self.num_vals());
|
||||
let from_value = *value_range.start();
|
||||
let to_value = *value_range.end();
|
||||
let from_value = *range.start();
|
||||
let to_value = *range.end();
|
||||
assert!(to_value >= from_value);
|
||||
let compact_from = self.u128_to_compact(from_value);
|
||||
let compact_to = self.u128_to_compact(to_value);
|
||||
@@ -367,7 +353,7 @@ impl CompactSpaceDecompressor {
|
||||
// Quick return, if both ranges fall into the same non-mapped space, the range can't cover
|
||||
// any values, so we can early exit
|
||||
match (compact_to, compact_from) {
|
||||
(Err(pos1), Err(pos2)) if pos1 == pos2 => return,
|
||||
(Err(pos1), Err(pos2)) if pos1 == pos2 => return Vec::new(),
|
||||
_ => {}
|
||||
}
|
||||
|
||||
@@ -389,28 +375,27 @@ impl CompactSpaceDecompressor {
|
||||
});
|
||||
|
||||
let range = compact_from..=compact_to;
|
||||
|
||||
let scan_num_docs = doc_id_range.end - doc_id_range.start;
|
||||
let mut positions = Vec::new();
|
||||
|
||||
let step_size = 4;
|
||||
let cutoff = doc_id_range.start + scan_num_docs - scan_num_docs % step_size;
|
||||
let cutoff = self.params.num_vals - self.params.num_vals % step_size;
|
||||
|
||||
let mut push_if_in_range = |idx, val| {
|
||||
if range.contains(&val) {
|
||||
positions.push(idx);
|
||||
}
|
||||
};
|
||||
let get_val = |idx| self.params.bit_unpacker.get(idx, &self.data);
|
||||
let get_val = |idx| self.params.bit_unpacker.get(idx as u64, &self.data);
|
||||
// unrolled loop
|
||||
for idx in (doc_id_range.start..cutoff).step_by(step_size as usize) {
|
||||
for idx in (0..cutoff).step_by(step_size as usize) {
|
||||
let idx1 = idx;
|
||||
let idx2 = idx + 1;
|
||||
let idx3 = idx + 2;
|
||||
let idx4 = idx + 3;
|
||||
let val1 = get_val(idx1 as u32);
|
||||
let val2 = get_val(idx2 as u32);
|
||||
let val3 = get_val(idx3 as u32);
|
||||
let val4 = get_val(idx4 as u32);
|
||||
let val1 = get_val(idx1);
|
||||
let val2 = get_val(idx2);
|
||||
let val3 = get_val(idx3);
|
||||
let val4 = get_val(idx4);
|
||||
push_if_in_range(idx1, val1);
|
||||
push_if_in_range(idx2, val2);
|
||||
push_if_in_range(idx3, val3);
|
||||
@@ -418,15 +403,17 @@ impl CompactSpaceDecompressor {
|
||||
}
|
||||
|
||||
// handle rest
|
||||
for idx in cutoff..doc_id_range.end {
|
||||
push_if_in_range(idx, get_val(idx as u32));
|
||||
for idx in cutoff..self.params.num_vals {
|
||||
push_if_in_range(idx, get_val(idx));
|
||||
}
|
||||
|
||||
positions
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn iter_compact(&self) -> impl Iterator<Item = u64> + '_ {
|
||||
(0..self.params.num_vals)
|
||||
.map(move |idx| self.params.bit_unpacker.get(idx, &self.data) as u64)
|
||||
.map(move |idx| self.params.bit_unpacker.get(idx as u64, &self.data) as u64)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
@@ -438,7 +425,7 @@ impl CompactSpaceDecompressor {
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn get(&self, idx: u32) -> u128 {
|
||||
pub fn get(&self, idx: u64) -> u128 {
|
||||
let compact = self.params.bit_unpacker.get(idx, &self.data);
|
||||
self.compact_to_u128(compact)
|
||||
}
|
||||
@@ -456,7 +443,7 @@ impl CompactSpaceDecompressor {
|
||||
mod tests {
|
||||
|
||||
use super::*;
|
||||
use crate::{open_u128, serialize_u128};
|
||||
use crate::{open_u128, serialize_u128, VecColumn};
|
||||
|
||||
#[test]
|
||||
fn compact_space_test() {
|
||||
@@ -465,7 +452,7 @@ mod tests {
|
||||
]
|
||||
.into_iter()
|
||||
.collect();
|
||||
let compact_space = get_compact_space(ips, ips.len() as u32, 11);
|
||||
let compact_space = get_compact_space(ips, ips.len() as u64, 11);
|
||||
let amplitude = compact_space.amplitude_compact_space();
|
||||
assert_eq!(amplitude, 17);
|
||||
assert_eq!(1, compact_space.u128_to_compact(2).unwrap());
|
||||
@@ -496,7 +483,7 @@ mod tests {
|
||||
#[test]
|
||||
fn compact_space_amplitude_test() {
|
||||
let ips = &[100000u128, 1000000].into_iter().collect();
|
||||
let compact_space = get_compact_space(ips, ips.len() as u32, 1);
|
||||
let compact_space = get_compact_space(ips, ips.len() as u64, 1);
|
||||
let amplitude = compact_space.amplitude_compact_space();
|
||||
assert_eq!(amplitude, 2);
|
||||
}
|
||||
@@ -504,21 +491,16 @@ mod tests {
|
||||
fn test_all(data: OwnedBytes, expected: &[u128]) {
|
||||
let decompressor = CompactSpaceDecompressor::open(data).unwrap();
|
||||
for (idx, expected_val) in expected.iter().cloned().enumerate() {
|
||||
let val = decompressor.get(idx as u32);
|
||||
let val = decompressor.get(idx as u64);
|
||||
assert_eq!(val, expected_val);
|
||||
|
||||
let test_range = |range: RangeInclusive<u128>| {
|
||||
let expected_positions = expected
|
||||
.iter()
|
||||
.positions(|val| range.contains(val))
|
||||
.map(|pos| pos as u32)
|
||||
.map(|pos| pos as u64)
|
||||
.collect::<Vec<_>>();
|
||||
let mut positions = Vec::new();
|
||||
decompressor.get_positions_for_value_range(
|
||||
range,
|
||||
0..decompressor.num_vals(),
|
||||
&mut positions,
|
||||
);
|
||||
let positions = decompressor.get_between_vals(range);
|
||||
assert_eq!(positions, expected_positions);
|
||||
};
|
||||
|
||||
@@ -531,12 +513,7 @@ mod tests {
|
||||
|
||||
fn test_aux_vals(u128_vals: &[u128]) -> OwnedBytes {
|
||||
let mut out = Vec::new();
|
||||
serialize_u128(
|
||||
|| u128_vals.iter().cloned(),
|
||||
u128_vals.len() as u32,
|
||||
&mut out,
|
||||
)
|
||||
.unwrap();
|
||||
serialize_u128(VecColumn::from(u128_vals), &mut out).unwrap();
|
||||
|
||||
let data = OwnedBytes::new(out);
|
||||
test_all(data.clone(), u128_vals);
|
||||
@@ -558,107 +535,24 @@ mod tests {
|
||||
];
|
||||
let data = test_aux_vals(vals);
|
||||
let decomp = CompactSpaceDecompressor::open(data).unwrap();
|
||||
let complete_range = 0..vals.len() as u32;
|
||||
for (pos, val) in vals.iter().enumerate() {
|
||||
let val = *val as u128;
|
||||
let pos = pos as u32;
|
||||
let mut positions = Vec::new();
|
||||
decomp.get_positions_for_value_range(val..=val, pos..pos + 1, &mut positions);
|
||||
assert_eq!(positions, vec![pos]);
|
||||
}
|
||||
|
||||
// handle docid range out of bounds
|
||||
let positions = get_positions_for_value_range_helper(&decomp, 0..=1, 1..u32::MAX);
|
||||
assert_eq!(positions, vec![]);
|
||||
|
||||
let positions =
|
||||
get_positions_for_value_range_helper(&decomp, 0..=1, complete_range.clone());
|
||||
let positions = decomp.get_between_vals(0..=1);
|
||||
assert_eq!(positions, vec![0]);
|
||||
let positions =
|
||||
get_positions_for_value_range_helper(&decomp, 0..=2, complete_range.clone());
|
||||
let positions = decomp.get_between_vals(0..=2);
|
||||
assert_eq!(positions, vec![0]);
|
||||
let positions =
|
||||
get_positions_for_value_range_helper(&decomp, 0..=3, complete_range.clone());
|
||||
let positions = decomp.get_between_vals(0..=3);
|
||||
assert_eq!(positions, vec![0, 2]);
|
||||
assert_eq!(
|
||||
get_positions_for_value_range_helper(
|
||||
&decomp,
|
||||
99999u128..=99999u128,
|
||||
complete_range.clone()
|
||||
),
|
||||
vec![3]
|
||||
);
|
||||
assert_eq!(
|
||||
get_positions_for_value_range_helper(
|
||||
&decomp,
|
||||
99999u128..=100000u128,
|
||||
complete_range.clone()
|
||||
),
|
||||
vec![3, 4]
|
||||
);
|
||||
assert_eq!(
|
||||
get_positions_for_value_range_helper(
|
||||
&decomp,
|
||||
99998u128..=100000u128,
|
||||
complete_range.clone()
|
||||
),
|
||||
vec![3, 4]
|
||||
);
|
||||
assert_eq!(
|
||||
get_positions_for_value_range_helper(
|
||||
&decomp,
|
||||
99998u128..=99999u128,
|
||||
complete_range.clone()
|
||||
),
|
||||
vec![3]
|
||||
);
|
||||
assert_eq!(
|
||||
get_positions_for_value_range_helper(
|
||||
&decomp,
|
||||
99998u128..=99998u128,
|
||||
complete_range.clone()
|
||||
),
|
||||
vec![]
|
||||
);
|
||||
assert_eq!(
|
||||
get_positions_for_value_range_helper(
|
||||
&decomp,
|
||||
333u128..=333u128,
|
||||
complete_range.clone()
|
||||
),
|
||||
vec![8]
|
||||
);
|
||||
assert_eq!(
|
||||
get_positions_for_value_range_helper(
|
||||
&decomp,
|
||||
332u128..=333u128,
|
||||
complete_range.clone()
|
||||
),
|
||||
vec![8]
|
||||
);
|
||||
assert_eq!(
|
||||
get_positions_for_value_range_helper(
|
||||
&decomp,
|
||||
332u128..=334u128,
|
||||
complete_range.clone()
|
||||
),
|
||||
vec![8]
|
||||
);
|
||||
assert_eq!(
|
||||
get_positions_for_value_range_helper(
|
||||
&decomp,
|
||||
333u128..=334u128,
|
||||
complete_range.clone()
|
||||
),
|
||||
vec![8]
|
||||
);
|
||||
assert_eq!(decomp.get_between_vals(99999u128..=99999u128), vec![3]);
|
||||
assert_eq!(decomp.get_between_vals(99999u128..=100000u128), vec![3, 4]);
|
||||
assert_eq!(decomp.get_between_vals(99998u128..=100000u128), vec![3, 4]);
|
||||
assert_eq!(decomp.get_between_vals(99998u128..=99999u128), vec![3]);
|
||||
assert_eq!(decomp.get_between_vals(99998u128..=99998u128), vec![]);
|
||||
assert_eq!(decomp.get_between_vals(333u128..=333u128), vec![8]);
|
||||
assert_eq!(decomp.get_between_vals(332u128..=333u128), vec![8]);
|
||||
assert_eq!(decomp.get_between_vals(332u128..=334u128), vec![8]);
|
||||
assert_eq!(decomp.get_between_vals(333u128..=334u128), vec![8]);
|
||||
|
||||
assert_eq!(
|
||||
get_positions_for_value_range_helper(
|
||||
&decomp,
|
||||
4_000_211_221u128..=5_000_000_000u128,
|
||||
complete_range.clone()
|
||||
),
|
||||
decomp.get_between_vals(4_000_211_221u128..=5_000_000_000u128),
|
||||
vec![6, 7]
|
||||
);
|
||||
}
|
||||
@@ -683,29 +577,12 @@ mod tests {
|
||||
];
|
||||
let data = test_aux_vals(vals);
|
||||
let decomp = CompactSpaceDecompressor::open(data).unwrap();
|
||||
let complete_range = 0..vals.len() as u32;
|
||||
assert_eq!(
|
||||
get_positions_for_value_range_helper(&decomp, 0..=5, complete_range.clone()),
|
||||
vec![]
|
||||
);
|
||||
assert_eq!(
|
||||
get_positions_for_value_range_helper(&decomp, 0..=100, complete_range.clone()),
|
||||
vec![0]
|
||||
);
|
||||
assert_eq!(
|
||||
get_positions_for_value_range_helper(&decomp, 0..=105, complete_range.clone()),
|
||||
vec![0]
|
||||
);
|
||||
}
|
||||
|
||||
fn get_positions_for_value_range_helper<C: Column<T> + ?Sized, T: PartialOrd>(
|
||||
column: &C,
|
||||
value_range: RangeInclusive<T>,
|
||||
doc_id_range: Range<u32>,
|
||||
) -> Vec<u32> {
|
||||
let mut positions = Vec::new();
|
||||
column.get_positions_for_value_range(value_range, doc_id_range, &mut positions);
|
||||
positions
|
||||
let positions = decomp.get_between_vals(0..=5);
|
||||
assert_eq!(positions, vec![]);
|
||||
let positions = decomp.get_between_vals(0..=100);
|
||||
assert_eq!(positions, vec![0]);
|
||||
let positions = decomp.get_between_vals(0..=105);
|
||||
assert_eq!(positions, vec![0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -726,33 +603,13 @@ mod tests {
|
||||
5_000_000_000,
|
||||
];
|
||||
let mut out = Vec::new();
|
||||
serialize_u128(|| vals.iter().cloned(), vals.len() as u32, &mut out).unwrap();
|
||||
let decomp = open_u128::<u128>(OwnedBytes::new(out)).unwrap();
|
||||
let complete_range = 0..vals.len() as u32;
|
||||
serialize_u128(VecColumn::from(vals), &mut out).unwrap();
|
||||
let decomp = open_u128(OwnedBytes::new(out)).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
get_positions_for_value_range_helper(&*decomp, 199..=200, complete_range.clone()),
|
||||
vec![0]
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
get_positions_for_value_range_helper(&*decomp, 199..=201, complete_range.clone()),
|
||||
vec![0, 1]
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
get_positions_for_value_range_helper(&*decomp, 200..=200, complete_range.clone()),
|
||||
vec![0]
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
get_positions_for_value_range_helper(
|
||||
&*decomp,
|
||||
1_000_000..=1_000_000,
|
||||
complete_range.clone()
|
||||
),
|
||||
vec![11]
|
||||
);
|
||||
assert_eq!(decomp.get_between_vals(199..=200), vec![0]);
|
||||
assert_eq!(decomp.get_between_vals(199..=201), vec![0, 1]);
|
||||
assert_eq!(decomp.get_between_vals(200..=200), vec![0]);
|
||||
assert_eq!(decomp.get_between_vals(1_000_000..=1_000_000), vec![11]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1,12 +1,5 @@
|
||||
#![warn(missing_docs)]
|
||||
#![cfg_attr(all(feature = "unstable", test), feature(test))]
|
||||
|
||||
//! # `fastfield_codecs`
|
||||
//!
|
||||
//! - Columnar storage of data for tantivy [`Column`].
|
||||
//! - Encode data in different codecs.
|
||||
//! - Monotonically map values to u64/u128
|
||||
|
||||
#[cfg(test)]
|
||||
#[macro_use]
|
||||
extern crate more_asserts;
|
||||
@@ -20,10 +13,6 @@ use std::sync::Arc;
|
||||
|
||||
use common::BinarySerializable;
|
||||
use compact_space::CompactSpaceDecompressor;
|
||||
use monotonic_mapping::{
|
||||
StrictlyMonotonicMappingInverter, StrictlyMonotonicMappingToInternal,
|
||||
StrictlyMonotonicMappingToInternalBaseval, StrictlyMonotonicMappingToInternalGCDBaseval,
|
||||
};
|
||||
use ownedbytes::OwnedBytes;
|
||||
use serialize::Header;
|
||||
|
||||
@@ -33,7 +22,6 @@ mod compact_space;
|
||||
mod line;
|
||||
mod linear;
|
||||
mod monotonic_mapping;
|
||||
mod monotonic_mapping_u128;
|
||||
|
||||
mod column;
|
||||
mod gcd;
|
||||
@@ -43,24 +31,16 @@ use self::bitpacked::BitpackedCodec;
|
||||
use self::blockwise_linear::BlockwiseLinearCodec;
|
||||
pub use self::column::{monotonic_map_column, Column, VecColumn};
|
||||
use self::linear::LinearCodec;
|
||||
pub use self::monotonic_mapping::{MonotonicallyMappableToU64, StrictlyMonotonicFn};
|
||||
pub use self::monotonic_mapping_u128::MonotonicallyMappableToU128;
|
||||
pub use self::monotonic_mapping::MonotonicallyMappableToU64;
|
||||
pub use self::serialize::{
|
||||
estimate, serialize, serialize_and_load, serialize_u128, NormalizedHeader,
|
||||
};
|
||||
|
||||
#[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)]
|
||||
#[repr(u8)]
|
||||
/// Available codecs to use to encode the u64 (via [`MonotonicallyMappableToU64`]) converted data.
|
||||
pub enum FastFieldCodecType {
|
||||
/// Bitpack all values in the value range. The number of bits is defined by the amplitude
|
||||
/// `column.max_value() - column.min_value()`
|
||||
Bitpacked = 1,
|
||||
/// Linear interpolation puts a line between the first and last value and then bitpacks the
|
||||
/// values by the offset from the line. The number of bits is defined by the max deviation from
|
||||
/// the line.
|
||||
Linear = 2,
|
||||
/// Same as [`FastFieldCodecType::Linear`], but encodes in blocks of 512 elements.
|
||||
BlockwiseLinear = 3,
|
||||
}
|
||||
|
||||
@@ -78,11 +58,11 @@ impl BinarySerializable for FastFieldCodecType {
|
||||
}
|
||||
|
||||
impl FastFieldCodecType {
|
||||
pub(crate) fn to_code(self) -> u8 {
|
||||
pub fn to_code(self) -> u8 {
|
||||
self as u8
|
||||
}
|
||||
|
||||
pub(crate) fn from_code(code: u8) -> Option<Self> {
|
||||
pub fn from_code(code: u8) -> Option<Self> {
|
||||
match code {
|
||||
1 => Some(Self::Bitpacked),
|
||||
2 => Some(Self::Linear),
|
||||
@@ -93,13 +73,8 @@ impl FastFieldCodecType {
|
||||
}
|
||||
|
||||
/// Returns the correct codec reader wrapped in the `Arc` for the data.
|
||||
pub fn open_u128<Item: MonotonicallyMappableToU128>(
|
||||
bytes: OwnedBytes,
|
||||
) -> io::Result<Arc<dyn Column<Item>>> {
|
||||
let reader = CompactSpaceDecompressor::open(bytes)?;
|
||||
let inverted: StrictlyMonotonicMappingInverter<StrictlyMonotonicMappingToInternal<Item>> =
|
||||
StrictlyMonotonicMappingToInternal::<Item>::new().into();
|
||||
Ok(Arc::new(monotonic_map_column(reader, inverted)))
|
||||
pub fn open_u128(bytes: OwnedBytes) -> io::Result<Arc<dyn Column<u128>>> {
|
||||
Ok(Arc::new(CompactSpaceDecompressor::open(bytes)?))
|
||||
}
|
||||
|
||||
/// Returns the correct codec reader wrapped in the `Arc` for the data.
|
||||
@@ -124,15 +99,11 @@ fn open_specific_codec<C: FastFieldCodec, Item: MonotonicallyMappableToU64>(
|
||||
let reader = C::open_from_bytes(bytes, normalized_header)?;
|
||||
let min_value = header.min_value;
|
||||
if let Some(gcd) = header.gcd {
|
||||
let mapping = StrictlyMonotonicMappingInverter::from(
|
||||
StrictlyMonotonicMappingToInternalGCDBaseval::new(gcd.get(), min_value),
|
||||
);
|
||||
Ok(Arc::new(monotonic_map_column(reader, mapping)))
|
||||
let monotonic_mapping = move |val: u64| Item::from_u64(min_value + val * gcd.get());
|
||||
Ok(Arc::new(monotonic_map_column(reader, monotonic_mapping)))
|
||||
} else {
|
||||
let mapping = StrictlyMonotonicMappingInverter::from(
|
||||
StrictlyMonotonicMappingToInternalBaseval::new(min_value),
|
||||
);
|
||||
Ok(Arc::new(monotonic_map_column(reader, mapping)))
|
||||
let monotonic_mapping = move |val: u64| Item::from_u64(min_value + val);
|
||||
Ok(Arc::new(monotonic_map_column(reader, monotonic_mapping)))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -164,7 +135,6 @@ trait FastFieldCodec: 'static {
|
||||
fn estimate(column: &dyn Column) -> Option<f32>;
|
||||
}
|
||||
|
||||
/// The list of all available codecs for u64 convertible data.
|
||||
pub const ALL_CODEC_TYPES: [FastFieldCodecType; 3] = [
|
||||
FastFieldCodecType::Bitpacked,
|
||||
FastFieldCodecType::BlockwiseLinear,
|
||||
@@ -173,7 +143,6 @@ pub const ALL_CODEC_TYPES: [FastFieldCodecType; 3] = [
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
use proptest::prelude::*;
|
||||
use proptest::strategy::Strategy;
|
||||
use proptest::{prop_oneof, proptest};
|
||||
@@ -199,32 +168,15 @@ mod tests {
|
||||
let actual_compression = out.len() as f32 / (data.len() as f32 * 8.0);
|
||||
|
||||
let reader = crate::open::<u64>(OwnedBytes::new(out)).unwrap();
|
||||
assert_eq!(reader.num_vals(), data.len() as u32);
|
||||
assert_eq!(reader.num_vals(), data.len() as u64);
|
||||
for (doc, orig_val) in data.iter().copied().enumerate() {
|
||||
let val = reader.get_val(doc as u32);
|
||||
let val = reader.get_val(doc as u64);
|
||||
assert_eq!(
|
||||
val, orig_val,
|
||||
"val `{val}` does not match orig_val {orig_val:?}, in data set {name}, data \
|
||||
`{data:?}`",
|
||||
);
|
||||
}
|
||||
|
||||
if !data.is_empty() {
|
||||
let test_rand_idx = rand::thread_rng().gen_range(0..=data.len() - 1);
|
||||
let expected_positions: Vec<u32> = data
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, el)| **el == data[test_rand_idx])
|
||||
.map(|(pos, _)| pos as u32)
|
||||
.collect();
|
||||
let mut positions = Vec::new();
|
||||
reader.get_positions_for_value_range(
|
||||
data[test_rand_idx]..=data[test_rand_idx],
|
||||
0..data.len() as u32,
|
||||
&mut positions,
|
||||
);
|
||||
assert_eq!(expected_positions, positions);
|
||||
}
|
||||
Some((estimation, actual_compression))
|
||||
}
|
||||
|
||||
@@ -434,7 +386,7 @@ mod bench {
|
||||
b.iter(|| {
|
||||
let mut sum = 0u64;
|
||||
for pos in value_iter() {
|
||||
let val = col.get_val(pos as u32);
|
||||
let val = col.get_val(pos as u64);
|
||||
sum = sum.wrapping_add(val);
|
||||
}
|
||||
sum
|
||||
@@ -446,7 +398,7 @@ mod bench {
|
||||
b.iter(|| {
|
||||
let mut sum = 0u64;
|
||||
for pos in value_iter() {
|
||||
let val = col.get_val(pos as u32);
|
||||
let val = col.get_val(pos as u64);
|
||||
sum = sum.wrapping_add(val);
|
||||
}
|
||||
sum
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use std::io;
|
||||
use std::num::NonZeroU32;
|
||||
use std::num::NonZeroU64;
|
||||
|
||||
use common::{BinarySerializable, VInt};
|
||||
|
||||
@@ -29,7 +29,7 @@ pub struct Line {
|
||||
/// compute_slope(y0, y1)
|
||||
/// = compute_slope(y0 + X % 2^64, y1 + X % 2^64)
|
||||
/// `
|
||||
fn compute_slope(y0: u64, y1: u64, num_vals: NonZeroU32) -> u64 {
|
||||
fn compute_slope(y0: u64, y1: u64, num_vals: NonZeroU64) -> u64 {
|
||||
let dy = y1.wrapping_sub(y0);
|
||||
let sign = dy <= (1 << 63);
|
||||
let abs_dy = if sign {
|
||||
@@ -43,7 +43,7 @@ fn compute_slope(y0: u64, y1: u64, num_vals: NonZeroU32) -> u64 {
|
||||
return 0u64;
|
||||
}
|
||||
|
||||
let abs_slope = (abs_dy << 32) / num_vals.get() as u64;
|
||||
let abs_slope = (abs_dy << 32) / num_vals.get();
|
||||
if sign {
|
||||
abs_slope
|
||||
} else {
|
||||
@@ -62,8 +62,8 @@ fn compute_slope(y0: u64, y1: u64, num_vals: NonZeroU32) -> u64 {
|
||||
|
||||
impl Line {
|
||||
#[inline(always)]
|
||||
pub fn eval(&self, x: u32) -> u64 {
|
||||
let linear_part = ((x as u64).wrapping_mul(self.slope) >> 32) as i32 as u64;
|
||||
pub fn eval(&self, x: u64) -> u64 {
|
||||
let linear_part = (x.wrapping_mul(self.slope) >> 32) as i32 as u64;
|
||||
self.intercept.wrapping_add(linear_part)
|
||||
}
|
||||
|
||||
@@ -75,7 +75,7 @@ impl Line {
|
||||
Self::train_from(
|
||||
first_val,
|
||||
last_val,
|
||||
num_vals as u32,
|
||||
num_vals,
|
||||
sample_positions_and_values.iter().cloned(),
|
||||
)
|
||||
}
|
||||
@@ -84,11 +84,11 @@ impl Line {
|
||||
fn train_from(
|
||||
first_val: u64,
|
||||
last_val: u64,
|
||||
num_vals: u32,
|
||||
num_vals: u64,
|
||||
positions_and_values: impl Iterator<Item = (u64, u64)>,
|
||||
) -> Self {
|
||||
// TODO replace with let else
|
||||
let idx_last_val = if let Some(idx_last_val) = NonZeroU32::new(num_vals - 1) {
|
||||
let idx_last_val = if let Some(idx_last_val) = NonZeroU64::new(num_vals - 1) {
|
||||
idx_last_val
|
||||
} else {
|
||||
return Line::default();
|
||||
@@ -129,7 +129,7 @@ impl Line {
|
||||
};
|
||||
let heuristic_shift = y0.wrapping_sub(MID_POINT);
|
||||
line.intercept = positions_and_values
|
||||
.map(|(pos, y)| y.wrapping_sub(line.eval(pos as u32)))
|
||||
.map(|(pos, y)| y.wrapping_sub(line.eval(pos)))
|
||||
.min_by_key(|&val| val.wrapping_sub(heuristic_shift))
|
||||
.unwrap_or(0u64); //< Never happens.
|
||||
line
|
||||
@@ -199,7 +199,7 @@ mod tests {
|
||||
let line = Line::train(&VecColumn::from(&ys));
|
||||
ys.iter()
|
||||
.enumerate()
|
||||
.map(|(x, y)| y.wrapping_sub(line.eval(x as u32)))
|
||||
.map(|(x, y)| y.wrapping_sub(line.eval(x as u64)))
|
||||
.max()
|
||||
}
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ pub struct LinearReader {
|
||||
|
||||
impl Column for LinearReader {
|
||||
#[inline]
|
||||
fn get_val(&self, doc: u32) -> u64 {
|
||||
fn get_val(&self, doc: u64) -> u64 {
|
||||
let interpoled_val: u64 = self.linear_params.line.eval(doc);
|
||||
let bitpacked_diff = self.linear_params.bit_unpacker.get(doc, &self.data);
|
||||
interpoled_val.wrapping_add(bitpacked_diff)
|
||||
@@ -37,7 +37,7 @@ impl Column for LinearReader {
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn num_vals(&self) -> u32 {
|
||||
fn num_vals(&self) -> u64 {
|
||||
self.header.num_vals
|
||||
}
|
||||
}
|
||||
@@ -93,7 +93,7 @@ impl FastFieldCodec for LinearCodec {
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(pos, actual_value)| {
|
||||
let calculated_value = line.eval(pos as u32);
|
||||
let calculated_value = line.eval(pos as u64);
|
||||
actual_value.wrapping_sub(calculated_value)
|
||||
})
|
||||
.max()
|
||||
@@ -108,7 +108,7 @@ impl FastFieldCodec for LinearCodec {
|
||||
|
||||
let mut bit_packer = BitPacker::new();
|
||||
for (pos, actual_value) in column.iter().enumerate() {
|
||||
let calculated_value = line.eval(pos as u32);
|
||||
let calculated_value = line.eval(pos as u64);
|
||||
let offset = actual_value.wrapping_sub(calculated_value);
|
||||
bit_packer.write(offset, num_bits, write)?;
|
||||
}
|
||||
@@ -140,7 +140,7 @@ impl FastFieldCodec for LinearCodec {
|
||||
let estimated_bit_width = sample_positions_and_values
|
||||
.into_iter()
|
||||
.map(|(pos, actual_value)| {
|
||||
let interpolated_val = line.eval(pos as u32);
|
||||
let interpolated_val = line.eval(pos as u64);
|
||||
actual_value.wrapping_sub(interpolated_val)
|
||||
})
|
||||
.map(|diff| ((diff as f32 * 1.5) * 2.0) as u64)
|
||||
|
||||
@@ -90,7 +90,7 @@ fn bench_ip() {
|
||||
{
|
||||
let mut data = vec![];
|
||||
for dataset in dataset.chunks(500_000) {
|
||||
serialize_u128(|| dataset.iter().cloned(), dataset.len() as u32, &mut data).unwrap();
|
||||
serialize_u128(VecColumn::from(dataset), &mut data).unwrap();
|
||||
}
|
||||
let compression = data.len() as f64 / (dataset.len() * 16) as f64;
|
||||
println!("Compression 50_000 chunks {:.4}", compression);
|
||||
@@ -101,10 +101,7 @@ fn bench_ip() {
|
||||
}
|
||||
|
||||
let mut data = vec![];
|
||||
{
|
||||
print_time!("creation");
|
||||
serialize_u128(|| dataset.iter().cloned(), dataset.len() as u32, &mut data).unwrap();
|
||||
}
|
||||
serialize_u128(VecColumn::from(&dataset), &mut data).unwrap();
|
||||
|
||||
let compression = data.len() as f64 / (dataset.len() * 16) as f64;
|
||||
println!("Compression {:.2}", compression);
|
||||
@@ -113,17 +110,11 @@ fn bench_ip() {
|
||||
(data.len() * 8) as f32 / dataset.len() as f32
|
||||
);
|
||||
|
||||
let decompressor = open_u128::<u128>(OwnedBytes::new(data)).unwrap();
|
||||
let decompressor = open_u128(OwnedBytes::new(data)).unwrap();
|
||||
// Sample some ranges
|
||||
let mut doc_values = Vec::new();
|
||||
for value in dataset.iter().take(1110).skip(1100).cloned() {
|
||||
doc_values.clear();
|
||||
print_time!("get range");
|
||||
decompressor.get_positions_for_value_range(
|
||||
value..=value,
|
||||
0..decompressor.num_vals(),
|
||||
&mut doc_values,
|
||||
);
|
||||
let doc_values = decompressor.get_between_vals(value..=value);
|
||||
println!("{:?}", doc_values.len());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,3 @@
|
||||
use std::marker::PhantomData;
|
||||
|
||||
use fastdivide::DividerU64;
|
||||
|
||||
use crate::MonotonicallyMappableToU128;
|
||||
|
||||
/// Monotonic maps a value to u64 value space.
|
||||
/// Monotonic mapping enables `PartialOrd` on u64 space without conversion to original space.
|
||||
pub trait MonotonicallyMappableToU64: 'static + PartialOrd + Copy + Send + Sync {
|
||||
/// Converts a value to u64.
|
||||
///
|
||||
@@ -19,145 +11,6 @@ pub trait MonotonicallyMappableToU64: 'static + PartialOrd + Copy + Send + Sync
|
||||
fn from_u64(val: u64) -> Self;
|
||||
}
|
||||
|
||||
/// Values need to be strictly monotonic mapped to a `Internal` value (u64 or u128) that can be
|
||||
/// used in fast field codecs.
|
||||
///
|
||||
/// The monotonic mapping is required so that `PartialOrd` can be used on `Internal` without
|
||||
/// converting to `External`.
|
||||
///
|
||||
/// All strictly monotonic functions are invertible because they are guaranteed to have a one-to-one
|
||||
/// mapping from their range to their domain. The `inverse` method is required when opening a codec,
|
||||
/// so a value can be converted back to its original domain (e.g. ip address or f64) from its
|
||||
/// internal representation.
|
||||
pub trait StrictlyMonotonicFn<External, Internal> {
|
||||
/// Strictly monotonically maps the value from External to Internal.
|
||||
fn mapping(&self, inp: External) -> Internal;
|
||||
/// Inverse of `mapping`. Maps the value from Internal to External.
|
||||
fn inverse(&self, out: Internal) -> External;
|
||||
}
|
||||
|
||||
/// Inverts a strictly monotonic mapping from `StrictlyMonotonicFn<A, B>` to
|
||||
/// `StrictlyMonotonicFn<B, A>`.
|
||||
///
|
||||
/// # Warning
|
||||
///
|
||||
/// This type comes with a footgun. A type being strictly monotonic does not impose that the inverse
|
||||
/// mapping is strictly monotonic over the entire space External. e.g. a -> a * 2. Use at your own
|
||||
/// risks.
|
||||
pub(crate) struct StrictlyMonotonicMappingInverter<T> {
|
||||
orig_mapping: T,
|
||||
}
|
||||
impl<T> From<T> for StrictlyMonotonicMappingInverter<T> {
|
||||
fn from(orig_mapping: T) -> Self {
|
||||
Self { orig_mapping }
|
||||
}
|
||||
}
|
||||
|
||||
impl<From, To, T> StrictlyMonotonicFn<To, From> for StrictlyMonotonicMappingInverter<T>
|
||||
where T: StrictlyMonotonicFn<From, To>
|
||||
{
|
||||
fn mapping(&self, val: To) -> From {
|
||||
self.orig_mapping.inverse(val)
|
||||
}
|
||||
|
||||
fn inverse(&self, val: From) -> To {
|
||||
self.orig_mapping.mapping(val)
|
||||
}
|
||||
}
|
||||
|
||||
/// Applies the strictly monotonic mapping from `T` without any additional changes.
|
||||
pub(crate) struct StrictlyMonotonicMappingToInternal<T> {
|
||||
_phantom: PhantomData<T>,
|
||||
}
|
||||
|
||||
impl<T> StrictlyMonotonicMappingToInternal<T> {
|
||||
pub(crate) fn new() -> StrictlyMonotonicMappingToInternal<T> {
|
||||
Self {
|
||||
_phantom: PhantomData,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<External: MonotonicallyMappableToU128, T: MonotonicallyMappableToU128>
|
||||
StrictlyMonotonicFn<External, u128> for StrictlyMonotonicMappingToInternal<T>
|
||||
where T: MonotonicallyMappableToU128
|
||||
{
|
||||
fn mapping(&self, inp: External) -> u128 {
|
||||
External::to_u128(inp)
|
||||
}
|
||||
|
||||
fn inverse(&self, out: u128) -> External {
|
||||
External::from_u128(out)
|
||||
}
|
||||
}
|
||||
|
||||
impl<External: MonotonicallyMappableToU64, T: MonotonicallyMappableToU64>
|
||||
StrictlyMonotonicFn<External, u64> for StrictlyMonotonicMappingToInternal<T>
|
||||
where T: MonotonicallyMappableToU64
|
||||
{
|
||||
fn mapping(&self, inp: External) -> u64 {
|
||||
External::to_u64(inp)
|
||||
}
|
||||
|
||||
fn inverse(&self, out: u64) -> External {
|
||||
External::from_u64(out)
|
||||
}
|
||||
}
|
||||
|
||||
/// Mapping dividing by gcd and a base value.
|
||||
///
|
||||
/// The function is assumed to be only called on values divided by passed
|
||||
/// gcd value. (It is necessary for the function to be monotonic.)
|
||||
pub(crate) struct StrictlyMonotonicMappingToInternalGCDBaseval {
|
||||
gcd_divider: DividerU64,
|
||||
gcd: u64,
|
||||
min_value: u64,
|
||||
}
|
||||
impl StrictlyMonotonicMappingToInternalGCDBaseval {
|
||||
pub(crate) fn new(gcd: u64, min_value: u64) -> Self {
|
||||
let gcd_divider = DividerU64::divide_by(gcd);
|
||||
Self {
|
||||
gcd_divider,
|
||||
gcd,
|
||||
min_value,
|
||||
}
|
||||
}
|
||||
}
|
||||
impl<External: MonotonicallyMappableToU64> StrictlyMonotonicFn<External, u64>
|
||||
for StrictlyMonotonicMappingToInternalGCDBaseval
|
||||
{
|
||||
fn mapping(&self, inp: External) -> u64 {
|
||||
self.gcd_divider
|
||||
.divide(External::to_u64(inp) - self.min_value)
|
||||
}
|
||||
|
||||
fn inverse(&self, out: u64) -> External {
|
||||
External::from_u64(self.min_value + out * self.gcd)
|
||||
}
|
||||
}
|
||||
|
||||
/// Strictly monotonic mapping with a base value.
|
||||
pub(crate) struct StrictlyMonotonicMappingToInternalBaseval {
|
||||
min_value: u64,
|
||||
}
|
||||
impl StrictlyMonotonicMappingToInternalBaseval {
|
||||
pub(crate) fn new(min_value: u64) -> Self {
|
||||
Self { min_value }
|
||||
}
|
||||
}
|
||||
|
||||
impl<External: MonotonicallyMappableToU64> StrictlyMonotonicFn<External, u64>
|
||||
for StrictlyMonotonicMappingToInternalBaseval
|
||||
{
|
||||
fn mapping(&self, val: External) -> u64 {
|
||||
External::to_u64(val) - self.min_value
|
||||
}
|
||||
|
||||
fn inverse(&self, val: u64) -> External {
|
||||
External::from_u64(self.min_value + val)
|
||||
}
|
||||
}
|
||||
|
||||
impl MonotonicallyMappableToU64 for u64 {
|
||||
fn to_u64(self) -> u64 {
|
||||
self
|
||||
@@ -201,33 +54,3 @@ impl MonotonicallyMappableToU64 for f64 {
|
||||
common::u64_to_f64(val)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn strictly_monotonic_test() {
|
||||
// identity mapping
|
||||
test_round_trip(&StrictlyMonotonicMappingToInternal::<u64>::new(), 100u64);
|
||||
// round trip to i64
|
||||
test_round_trip(&StrictlyMonotonicMappingToInternal::<i64>::new(), 100u64);
|
||||
// identity mapping
|
||||
test_round_trip(&StrictlyMonotonicMappingToInternal::<u128>::new(), 100u128);
|
||||
|
||||
// base value to i64 round trip
|
||||
let mapping = StrictlyMonotonicMappingToInternalBaseval::new(100);
|
||||
test_round_trip::<_, _, u64>(&mapping, 100i64);
|
||||
// base value and gcd to u64 round trip
|
||||
let mapping = StrictlyMonotonicMappingToInternalGCDBaseval::new(10, 100);
|
||||
test_round_trip::<_, _, u64>(&mapping, 100u64);
|
||||
}
|
||||
|
||||
fn test_round_trip<T: StrictlyMonotonicFn<K, L>, K: std::fmt::Debug + Eq + Copy, L>(
|
||||
mapping: &T,
|
||||
test_val: K,
|
||||
) {
|
||||
assert_eq!(mapping.inverse(mapping.mapping(test_val)), test_val);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
use std::net::Ipv6Addr;
|
||||
use std::net::{IpAddr, Ipv6Addr};
|
||||
|
||||
/// Montonic maps a value to u128 value space
|
||||
/// Monotonic mapping enables `PartialOrd` on u128 space without conversion to original space.
|
||||
pub trait MonotonicallyMappableToU128: 'static + PartialOrd + Copy + Send + Sync {
|
||||
/// Converts a value to u128.
|
||||
///
|
||||
@@ -25,16 +23,20 @@ impl MonotonicallyMappableToU128 for u128 {
|
||||
}
|
||||
}
|
||||
|
||||
impl MonotonicallyMappableToU128 for Ipv6Addr {
|
||||
impl MonotonicallyMappableToU128 for IpAddr {
|
||||
fn to_u128(self) -> u128 {
|
||||
ip_to_u128(self)
|
||||
}
|
||||
|
||||
fn from_u128(val: u128) -> Self {
|
||||
Ipv6Addr::from(val.to_be_bytes())
|
||||
IpAddr::from(val.to_be_bytes())
|
||||
}
|
||||
}
|
||||
|
||||
fn ip_to_u128(ip_addr: Ipv6Addr) -> u128 {
|
||||
u128::from_be_bytes(ip_addr.octets())
|
||||
fn ip_to_u128(ip_addr: IpAddr) -> u128 {
|
||||
let ip_addr_v6: Ipv6Addr = match ip_addr {
|
||||
IpAddr::V4(v4) => v4.to_ipv6_mapped(),
|
||||
IpAddr::V6(v6) => v6,
|
||||
};
|
||||
u128::from_be_bytes(ip_addr_v6.octets())
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ use std::num::NonZeroU64;
|
||||
use std::sync::Arc;
|
||||
|
||||
use common::{BinarySerializable, VInt};
|
||||
use fastdivide::DividerU64;
|
||||
use log::warn;
|
||||
use ownedbytes::OwnedBytes;
|
||||
|
||||
@@ -29,10 +30,6 @@ use crate::bitpacked::BitpackedCodec;
|
||||
use crate::blockwise_linear::BlockwiseLinearCodec;
|
||||
use crate::compact_space::CompactSpaceCompressor;
|
||||
use crate::linear::LinearCodec;
|
||||
use crate::monotonic_mapping::{
|
||||
StrictlyMonotonicFn, StrictlyMonotonicMappingToInternal,
|
||||
StrictlyMonotonicMappingToInternalGCDBaseval,
|
||||
};
|
||||
use crate::{
|
||||
monotonic_map_column, Column, FastFieldCodec, FastFieldCodecType, MonotonicallyMappableToU64,
|
||||
VecColumn, ALL_CODEC_TYPES,
|
||||
@@ -40,20 +37,18 @@ use crate::{
|
||||
|
||||
/// The normalized header gives some parameters after applying the following
|
||||
/// normalization of the vector:
|
||||
/// `val -> (val - min_value) / gcd`
|
||||
/// val -> (val - min_value) / gcd
|
||||
///
|
||||
/// By design, after normalization, `min_value = 0` and `gcd = 1`.
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
pub struct NormalizedHeader {
|
||||
/// The number of values in the underlying column.
|
||||
pub num_vals: u32,
|
||||
/// The max value of the underlying column.
|
||||
pub num_vals: u64,
|
||||
pub max_value: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone)]
|
||||
pub(crate) struct Header {
|
||||
pub num_vals: u32,
|
||||
pub num_vals: u64,
|
||||
pub min_value: u64,
|
||||
pub max_value: u64,
|
||||
pub gcd: Option<NonZeroU64>,
|
||||
@@ -62,11 +57,8 @@ pub(crate) struct Header {
|
||||
|
||||
impl Header {
|
||||
pub fn normalized(self) -> NormalizedHeader {
|
||||
let gcd = self.gcd.map(|gcd| gcd.get()).unwrap_or(1);
|
||||
let gcd_min_val_mapping =
|
||||
StrictlyMonotonicMappingToInternalGCDBaseval::new(gcd, self.min_value);
|
||||
|
||||
let max_value = gcd_min_val_mapping.mapping(self.max_value);
|
||||
let max_value =
|
||||
(self.max_value - self.min_value) / self.gcd.map(|gcd| gcd.get()).unwrap_or(1);
|
||||
NormalizedHeader {
|
||||
num_vals: self.num_vals,
|
||||
max_value,
|
||||
@@ -74,7 +66,10 @@ impl Header {
|
||||
}
|
||||
|
||||
pub fn normalize_column<C: Column>(&self, from_column: C) -> impl Column {
|
||||
normalize_column(from_column, self.min_value, self.gcd)
|
||||
let min_value = self.min_value;
|
||||
let gcd = self.gcd.map(|gcd| gcd.get()).unwrap_or(1);
|
||||
let divider = DividerU64::divide_by(gcd);
|
||||
monotonic_map_column(from_column, move |val| divider.divide(val - min_value))
|
||||
}
|
||||
|
||||
pub fn compute_header(
|
||||
@@ -86,8 +81,9 @@ impl Header {
|
||||
let max_value = column.max_value();
|
||||
let gcd = crate::gcd::find_gcd(column.iter().map(|val| val - min_value))
|
||||
.filter(|gcd| gcd.get() > 1u64);
|
||||
let normalized_column = normalize_column(column, min_value, gcd);
|
||||
let codec_type = detect_codec(normalized_column, codecs)?;
|
||||
let divider = DividerU64::divide_by(gcd.map(|gcd| gcd.get()).unwrap_or(1u64));
|
||||
let shifted_column = monotonic_map_column(&column, |val| divider.divide(val - min_value));
|
||||
let codec_type = detect_codec(shifted_column, codecs)?;
|
||||
Some(Header {
|
||||
num_vals,
|
||||
min_value,
|
||||
@@ -98,19 +94,9 @@ impl Header {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn normalize_column<C: Column>(
|
||||
from_column: C,
|
||||
min_value: u64,
|
||||
gcd: Option<NonZeroU64>,
|
||||
) -> impl Column {
|
||||
let gcd = gcd.map(|gcd| gcd.get()).unwrap_or(1);
|
||||
let mapping = StrictlyMonotonicMappingToInternalGCDBaseval::new(gcd, min_value);
|
||||
monotonic_map_column(from_column, mapping)
|
||||
}
|
||||
|
||||
impl BinarySerializable for Header {
|
||||
fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
|
||||
VInt(self.num_vals as u64).serialize(writer)?;
|
||||
VInt(self.num_vals).serialize(writer)?;
|
||||
VInt(self.min_value).serialize(writer)?;
|
||||
VInt(self.max_value - self.min_value).serialize(writer)?;
|
||||
if let Some(gcd) = self.gcd {
|
||||
@@ -123,7 +109,7 @@ impl BinarySerializable for Header {
|
||||
}
|
||||
|
||||
fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
|
||||
let num_vals = VInt::deserialize(reader)?.0 as u32;
|
||||
let num_vals = VInt::deserialize(reader)?.0;
|
||||
let min_value = VInt::deserialize(reader)?.0;
|
||||
let amplitude = VInt::deserialize(reader)?.0;
|
||||
let max_value = min_value + amplitude;
|
||||
@@ -139,21 +125,16 @@ impl BinarySerializable for Header {
|
||||
}
|
||||
}
|
||||
|
||||
/// Return estimated compression for given codec in the value range [0.0..1.0], where 1.0 means no
|
||||
/// compression.
|
||||
pub fn estimate<T: MonotonicallyMappableToU64>(
|
||||
typed_column: impl Column<T>,
|
||||
codec_type: FastFieldCodecType,
|
||||
) -> Option<f32> {
|
||||
let column = monotonic_map_column(typed_column, StrictlyMonotonicMappingToInternal::<T>::new());
|
||||
let column = monotonic_map_column(typed_column, T::to_u64);
|
||||
let min_value = column.min_value();
|
||||
let gcd = crate::gcd::find_gcd(column.iter().map(|val| val - min_value))
|
||||
.filter(|gcd| gcd.get() > 1u64);
|
||||
let mapping = StrictlyMonotonicMappingToInternalGCDBaseval::new(
|
||||
gcd.map(|gcd| gcd.get()).unwrap_or(1u64),
|
||||
min_value,
|
||||
);
|
||||
let normalized_column = monotonic_map_column(&column, mapping);
|
||||
let divider = DividerU64::divide_by(gcd.map(|gcd| gcd.get()).unwrap_or(1u64));
|
||||
let normalized_column = monotonic_map_column(&column, |val| divider.divide(val - min_value));
|
||||
match codec_type {
|
||||
FastFieldCodecType::Bitpacked => BitpackedCodec::estimate(&normalized_column),
|
||||
FastFieldCodecType::Linear => LinearCodec::estimate(&normalized_column),
|
||||
@@ -161,26 +142,25 @@ pub fn estimate<T: MonotonicallyMappableToU64>(
|
||||
}
|
||||
}
|
||||
|
||||
/// Serializes u128 values with the compact space codec.
|
||||
pub fn serialize_u128<F: Fn() -> I, I: Iterator<Item = u128>>(
|
||||
iter_gen: F,
|
||||
num_vals: u32,
|
||||
pub fn serialize_u128(
|
||||
typed_column: impl Column<u128>,
|
||||
output: &mut impl io::Write,
|
||||
) -> io::Result<()> {
|
||||
// TODO write header, to later support more codecs
|
||||
let compressor = CompactSpaceCompressor::train_from(iter_gen(), num_vals);
|
||||
compressor.compress_into(iter_gen(), output).unwrap();
|
||||
let compressor = CompactSpaceCompressor::train_from(&typed_column);
|
||||
compressor
|
||||
.compress_into(typed_column.iter(), output)
|
||||
.unwrap();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Serializes the column with the codec with the best estimate on the data.
|
||||
pub fn serialize<T: MonotonicallyMappableToU64>(
|
||||
typed_column: impl Column<T>,
|
||||
output: &mut impl io::Write,
|
||||
codecs: &[FastFieldCodecType],
|
||||
) -> io::Result<()> {
|
||||
let column = monotonic_map_column(typed_column, StrictlyMonotonicMappingToInternal::<T>::new());
|
||||
let column = monotonic_map_column(typed_column, T::to_u64);
|
||||
let header = Header::compute_header(&column, codecs).ok_or_else(|| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
@@ -245,7 +225,6 @@ fn serialize_given_codec(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Helper function to serialize a column (autodetect from all codecs) and then open it
|
||||
pub fn serialize_and_load<T: MonotonicallyMappableToU64 + Ord + Default>(
|
||||
column: &[T],
|
||||
) -> Arc<dyn Column<T>> {
|
||||
|
||||
@@ -62,20 +62,6 @@ fn word<'a>() -> impl Parser<&'a str, Output = String> {
|
||||
})
|
||||
}
|
||||
|
||||
// word variant that allows more characters, e.g. for range queries that don't allow field
|
||||
// specifier
|
||||
fn relaxed_word<'a>() -> impl Parser<&'a str, Output = String> {
|
||||
(
|
||||
satisfy(|c: char| {
|
||||
!c.is_whitespace() && !['`', '{', '}', '"', '[', ']', '(', ')'].contains(&c)
|
||||
}),
|
||||
many(satisfy(|c: char| {
|
||||
!c.is_whitespace() && !['{', '}', '"', '[', ']', '(', ')'].contains(&c)
|
||||
})),
|
||||
)
|
||||
.map(|(s1, s2): (char, String)| format!("{}{}", s1, s2))
|
||||
}
|
||||
|
||||
/// Parses a date time according to rfc3339
|
||||
/// 2015-08-02T18:54:42+02
|
||||
/// 2021-04-13T19:46:26.266051969+00:00
|
||||
@@ -195,8 +181,8 @@ fn spaces1<'a>() -> impl Parser<&'a str, Output = ()> {
|
||||
fn range<'a>() -> impl Parser<&'a str, Output = UserInputLeaf> {
|
||||
let range_term_val = || {
|
||||
attempt(date_time())
|
||||
.or(word())
|
||||
.or(negative_number())
|
||||
.or(relaxed_word())
|
||||
.or(char('*').with(value("*".to_string())))
|
||||
};
|
||||
|
||||
@@ -663,34 +649,6 @@ mod test {
|
||||
.expect("Cannot parse date range")
|
||||
.0;
|
||||
assert_eq!(res6, expected_flexible_dates);
|
||||
// IP Range Unbounded
|
||||
let expected_weight = UserInputLeaf::Range {
|
||||
field: Some("ip".to_string()),
|
||||
lower: UserInputBound::Inclusive("::1".to_string()),
|
||||
upper: UserInputBound::Unbounded,
|
||||
};
|
||||
let res1 = range()
|
||||
.parse("ip: >=::1")
|
||||
.expect("Cannot parse ip v6 format")
|
||||
.0;
|
||||
let res2 = range()
|
||||
.parse("ip:[::1 TO *}")
|
||||
.expect("Cannot parse ip v6 format")
|
||||
.0;
|
||||
assert_eq!(res1, expected_weight);
|
||||
assert_eq!(res2, expected_weight);
|
||||
|
||||
// IP Range Bounded
|
||||
let expected_weight = UserInputLeaf::Range {
|
||||
field: Some("ip".to_string()),
|
||||
lower: UserInputBound::Inclusive("::0.0.0.50".to_string()),
|
||||
upper: UserInputBound::Exclusive("::0.0.0.52".to_string()),
|
||||
};
|
||||
let res1 = range()
|
||||
.parse("ip:[::0.0.0.50 TO ::0.0.0.52}")
|
||||
.expect("Cannot parse ip v6 format")
|
||||
.0;
|
||||
assert_eq!(res1, expected_weight);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use rustc_hash::FxHashMap;
|
||||
use fnv::FnvHashMap;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::agg_req::BucketAggregationInternal;
|
||||
@@ -145,7 +145,7 @@ pub enum BucketEntries<T> {
|
||||
/// Vector format bucket entries
|
||||
Vec(Vec<T>),
|
||||
/// HashMap format bucket entries
|
||||
HashMap(FxHashMap<String, T>),
|
||||
HashMap(FnvHashMap<String, T>),
|
||||
}
|
||||
|
||||
/// This is the default entry for a bucket, which contains a key, count, and optionally
|
||||
|
||||
@@ -331,10 +331,10 @@ impl SegmentHistogramCollector {
|
||||
.expect("unexpected fast field cardinatility");
|
||||
let mut iter = doc.chunks_exact(4);
|
||||
for docs in iter.by_ref() {
|
||||
let val0 = self.f64_from_fastfield_u64(accessor.get_val(docs[0]));
|
||||
let val1 = self.f64_from_fastfield_u64(accessor.get_val(docs[1]));
|
||||
let val2 = self.f64_from_fastfield_u64(accessor.get_val(docs[2]));
|
||||
let val3 = self.f64_from_fastfield_u64(accessor.get_val(docs[3]));
|
||||
let val0 = self.f64_from_fastfield_u64(accessor.get_val(docs[0] as u64));
|
||||
let val1 = self.f64_from_fastfield_u64(accessor.get_val(docs[1] as u64));
|
||||
let val2 = self.f64_from_fastfield_u64(accessor.get_val(docs[2] as u64));
|
||||
let val3 = self.f64_from_fastfield_u64(accessor.get_val(docs[3] as u64));
|
||||
|
||||
let bucket_pos0 = get_bucket_num(val0);
|
||||
let bucket_pos1 = get_bucket_num(val1);
|
||||
@@ -371,7 +371,7 @@ impl SegmentHistogramCollector {
|
||||
)?;
|
||||
}
|
||||
for &doc in iter.remainder() {
|
||||
let val = f64_from_fastfield_u64(accessor.get_val(doc), &self.field_type);
|
||||
let val = f64_from_fastfield_u64(accessor.get_val(doc as u64), &self.field_type);
|
||||
if !bounds.contains(val) {
|
||||
continue;
|
||||
}
|
||||
@@ -452,7 +452,7 @@ fn intermediate_buckets_to_final_buckets_fill_gaps(
|
||||
histogram_req: &HistogramAggregation,
|
||||
sub_aggregation: &AggregationsInternal,
|
||||
) -> crate::Result<Vec<BucketEntry>> {
|
||||
// Generate the full list of buckets without gaps.
|
||||
// Generate the the full list of buckets without gaps.
|
||||
//
|
||||
// The bounds are the min max from the current buckets, optionally extended by
|
||||
// extended_bounds from the request
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use std::fmt::Debug;
|
||||
use std::ops::Range;
|
||||
|
||||
use rustc_hash::FxHashMap;
|
||||
use fnv::FnvHashMap;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::aggregation::agg_req_with_accessor::{
|
||||
@@ -176,7 +176,7 @@ impl SegmentRangeCollector {
|
||||
) -> crate::Result<IntermediateBucketResult> {
|
||||
let field_type = self.field_type;
|
||||
|
||||
let buckets: FxHashMap<SerializedKey, IntermediateRangeBucketEntry> = self
|
||||
let buckets: FnvHashMap<SerializedKey, IntermediateRangeBucketEntry> = self
|
||||
.buckets
|
||||
.into_iter()
|
||||
.map(move |range_bucket| {
|
||||
@@ -263,10 +263,10 @@ impl SegmentRangeCollector {
|
||||
.as_single()
|
||||
.expect("unexpected fast field cardinality");
|
||||
for docs in iter.by_ref() {
|
||||
let val1 = accessor.get_val(docs[0]);
|
||||
let val2 = accessor.get_val(docs[1]);
|
||||
let val3 = accessor.get_val(docs[2]);
|
||||
let val4 = accessor.get_val(docs[3]);
|
||||
let val1 = accessor.get_val(docs[0] as u64);
|
||||
let val2 = accessor.get_val(docs[1] as u64);
|
||||
let val3 = accessor.get_val(docs[2] as u64);
|
||||
let val4 = accessor.get_val(docs[3] as u64);
|
||||
let bucket_pos1 = self.get_bucket_pos(val1);
|
||||
let bucket_pos2 = self.get_bucket_pos(val2);
|
||||
let bucket_pos3 = self.get_bucket_pos(val3);
|
||||
@@ -278,7 +278,7 @@ impl SegmentRangeCollector {
|
||||
self.increment_bucket(bucket_pos4, docs[3], &bucket_with_accessor.sub_aggregation)?;
|
||||
}
|
||||
for &doc in iter.remainder() {
|
||||
let val = accessor.get_val(doc);
|
||||
let val = accessor.get_val(doc as u64);
|
||||
let bucket_pos = self.get_bucket_pos(val);
|
||||
self.increment_bucket(bucket_pos, doc, &bucket_with_accessor.sub_aggregation)?;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use std::fmt::Debug;
|
||||
|
||||
use fnv::FnvHashMap;
|
||||
use itertools::Itertools;
|
||||
use rustc_hash::FxHashMap;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{CustomOrder, Order, OrderTarget};
|
||||
@@ -17,11 +17,7 @@ use crate::fastfield::MultiValuedFastFieldReader;
|
||||
use crate::schema::Type;
|
||||
use crate::{DocId, TantivyError};
|
||||
|
||||
/// Creates a bucket for every unique term and counts the number of occurences.
|
||||
/// Note that doc_count in the response buckets equals term count here.
|
||||
///
|
||||
/// If the text is untokenized and single value, that means one term per document and therefore it
|
||||
/// is in fact doc count.
|
||||
/// Creates a bucket for every unique term
|
||||
///
|
||||
/// ### Terminology
|
||||
/// Shard parameters are supposed to be equivalent to elasticsearch shard parameter.
|
||||
@@ -68,25 +64,6 @@ use crate::{DocId, TantivyError};
|
||||
/// }
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// /// # Response JSON Format
|
||||
/// ```json
|
||||
/// {
|
||||
/// ...
|
||||
/// "aggregations": {
|
||||
/// "genres": {
|
||||
/// "doc_count_error_upper_bound": 0,
|
||||
/// "sum_other_doc_count": 0,
|
||||
/// "buckets": [
|
||||
/// { "key": "drumnbass", "doc_count": 6 },
|
||||
/// { "key": "raggae", "doc_count": 4 },
|
||||
/// { "key": "jazz", "doc_count": 2 }
|
||||
/// ]
|
||||
/// }
|
||||
/// }
|
||||
/// }
|
||||
/// ```
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct TermsAggregation {
|
||||
/// The field to aggregate on.
|
||||
@@ -199,7 +176,7 @@ impl TermsAggregationInternal {
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
/// Container to store term_ids and their buckets.
|
||||
struct TermBuckets {
|
||||
pub(crate) entries: FxHashMap<u32, TermBucketEntry>,
|
||||
pub(crate) entries: FnvHashMap<u32, TermBucketEntry>,
|
||||
blueprint: Option<SegmentAggregationResultsCollector>,
|
||||
}
|
||||
|
||||
@@ -397,7 +374,7 @@ impl SegmentTermCollector {
|
||||
.expect("internal error: inverted index not loaded for term aggregation");
|
||||
let term_dict = inverted_index.terms();
|
||||
|
||||
let mut dict: FxHashMap<String, IntermediateTermBucketEntry> = Default::default();
|
||||
let mut dict: FnvHashMap<String, IntermediateTermBucketEntry> = Default::default();
|
||||
let mut buffer = vec![];
|
||||
for (term_id, entry) in entries {
|
||||
term_dict
|
||||
@@ -1129,9 +1106,9 @@ mod tests {
|
||||
|
||||
assert_eq!(res["my_texts"]["buckets"][0]["key"], "terma");
|
||||
assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 4);
|
||||
assert_eq!(res["my_texts"]["buckets"][1]["key"], "termc");
|
||||
assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");
|
||||
assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 0);
|
||||
assert_eq!(res["my_texts"]["buckets"][2]["key"], "termb");
|
||||
assert_eq!(res["my_texts"]["buckets"][2]["key"], "termc");
|
||||
assert_eq!(res["my_texts"]["buckets"][2]["doc_count"], 0);
|
||||
assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);
|
||||
assert_eq!(res["my_texts"]["doc_count_error_upper_bound"], 0);
|
||||
@@ -1229,43 +1206,11 @@ mod tests {
|
||||
.collect();
|
||||
|
||||
let res = exec_request_with_query(agg_req, &index, None);
|
||||
|
||||
assert!(res.is_err());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn terms_aggregation_multi_token_per_doc() -> crate::Result<()> {
|
||||
let terms = vec!["Hello Hello", "Hallo Hallo"];
|
||||
|
||||
let index = get_test_index_from_terms(true, &[terms])?;
|
||||
|
||||
let agg_req: Aggregations = vec![(
|
||||
"my_texts".to_string(),
|
||||
Aggregation::Bucket(BucketAggregation {
|
||||
bucket_agg: BucketAggregationType::Terms(TermsAggregation {
|
||||
field: "text_id".to_string(),
|
||||
min_doc_count: Some(0),
|
||||
..Default::default()
|
||||
}),
|
||||
sub_aggregation: Default::default(),
|
||||
}),
|
||||
)]
|
||||
.into_iter()
|
||||
.collect();
|
||||
|
||||
let res = exec_request_with_query(agg_req, &index, None).unwrap();
|
||||
|
||||
assert_eq!(res["my_texts"]["buckets"][0]["key"], "hello");
|
||||
assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 2);
|
||||
|
||||
assert_eq!(res["my_texts"]["buckets"][1]["key"], "hallo");
|
||||
assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_json_format() -> crate::Result<()> {
|
||||
let agg_req: Aggregations = vec![(
|
||||
|
||||
@@ -5,8 +5,8 @@
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use fnv::FnvHashMap;
|
||||
use itertools::Itertools;
|
||||
use rustc_hash::FxHashMap;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::agg_req::{
|
||||
@@ -288,7 +288,7 @@ impl IntermediateBucketResult {
|
||||
.keyed;
|
||||
let buckets = if is_keyed {
|
||||
let mut bucket_map =
|
||||
FxHashMap::with_capacity_and_hasher(buckets.len(), Default::default());
|
||||
FnvHashMap::with_capacity_and_hasher(buckets.len(), Default::default());
|
||||
for bucket in buckets {
|
||||
bucket_map.insert(bucket.key.to_string(), bucket);
|
||||
}
|
||||
@@ -308,7 +308,7 @@ impl IntermediateBucketResult {
|
||||
|
||||
let buckets = if req.as_histogram().unwrap().keyed {
|
||||
let mut bucket_map =
|
||||
FxHashMap::with_capacity_and_hasher(buckets.len(), Default::default());
|
||||
FnvHashMap::with_capacity_and_hasher(buckets.len(), Default::default());
|
||||
for bucket in buckets {
|
||||
bucket_map.insert(bucket.key.to_string(), bucket);
|
||||
}
|
||||
@@ -396,13 +396,13 @@ impl IntermediateBucketResult {
|
||||
#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
/// Range aggregation including error counts
|
||||
pub struct IntermediateRangeBucketResult {
|
||||
pub(crate) buckets: FxHashMap<SerializedKey, IntermediateRangeBucketEntry>,
|
||||
pub(crate) buckets: FnvHashMap<SerializedKey, IntermediateRangeBucketEntry>,
|
||||
}
|
||||
|
||||
#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
/// Term aggregation including error counts
|
||||
pub struct IntermediateTermBucketResult {
|
||||
pub(crate) entries: FxHashMap<String, IntermediateTermBucketEntry>,
|
||||
pub(crate) entries: FnvHashMap<String, IntermediateTermBucketEntry>,
|
||||
pub(crate) sum_other_doc_count: u64,
|
||||
pub(crate) doc_count_error_upper_bound: u64,
|
||||
}
|
||||
@@ -499,8 +499,8 @@ trait MergeFruits {
|
||||
}
|
||||
|
||||
fn merge_maps<V: MergeFruits + Clone>(
|
||||
entries_left: &mut FxHashMap<SerializedKey, V>,
|
||||
mut entries_right: FxHashMap<SerializedKey, V>,
|
||||
entries_left: &mut FnvHashMap<SerializedKey, V>,
|
||||
mut entries_right: FnvHashMap<SerializedKey, V>,
|
||||
) {
|
||||
for (name, entry_left) in entries_left.iter_mut() {
|
||||
if let Some(entry_right) = entries_right.remove(name) {
|
||||
@@ -626,7 +626,7 @@ mod tests {
|
||||
|
||||
fn get_sub_test_tree(data: &[(String, u64)]) -> IntermediateAggregationResults {
|
||||
let mut map = HashMap::new();
|
||||
let mut buckets = FxHashMap::default();
|
||||
let mut buckets = FnvHashMap::default();
|
||||
for (key, doc_count) in data {
|
||||
buckets.insert(
|
||||
key.to_string(),
|
||||
@@ -653,7 +653,7 @@ mod tests {
|
||||
data: &[(String, u64, String, u64)],
|
||||
) -> IntermediateAggregationResults {
|
||||
let mut map = HashMap::new();
|
||||
let mut buckets: FxHashMap<_, _> = Default::default();
|
||||
let mut buckets: FnvHashMap<_, _> = Default::default();
|
||||
for (key, doc_count, sub_aggregation_key, sub_aggregation_count) in data {
|
||||
buckets.insert(
|
||||
key.to_string(),
|
||||
|
||||
@@ -60,10 +60,10 @@ impl SegmentAverageCollector {
|
||||
pub(crate) fn collect_block(&mut self, doc: &[DocId], field: &dyn Column<u64>) {
|
||||
let mut iter = doc.chunks_exact(4);
|
||||
for docs in iter.by_ref() {
|
||||
let val1 = field.get_val(docs[0]);
|
||||
let val2 = field.get_val(docs[1]);
|
||||
let val3 = field.get_val(docs[2]);
|
||||
let val4 = field.get_val(docs[3]);
|
||||
let val1 = field.get_val(docs[0] as u64);
|
||||
let val2 = field.get_val(docs[1] as u64);
|
||||
let val3 = field.get_val(docs[2] as u64);
|
||||
let val4 = field.get_val(docs[3] as u64);
|
||||
let val1 = f64_from_fastfield_u64(val1, &self.field_type);
|
||||
let val2 = f64_from_fastfield_u64(val2, &self.field_type);
|
||||
let val3 = f64_from_fastfield_u64(val3, &self.field_type);
|
||||
@@ -74,7 +74,7 @@ impl SegmentAverageCollector {
|
||||
self.data.collect(val4);
|
||||
}
|
||||
for &doc in iter.remainder() {
|
||||
let val = field.get_val(doc);
|
||||
let val = field.get_val(doc as u64);
|
||||
let val = f64_from_fastfield_u64(val, &self.field_type);
|
||||
self.data.collect(val);
|
||||
}
|
||||
|
||||
@@ -166,10 +166,10 @@ impl SegmentStatsCollector {
|
||||
pub(crate) fn collect_block(&mut self, doc: &[DocId], field: &dyn Column<u64>) {
|
||||
let mut iter = doc.chunks_exact(4);
|
||||
for docs in iter.by_ref() {
|
||||
let val1 = field.get_val(docs[0]);
|
||||
let val2 = field.get_val(docs[1]);
|
||||
let val3 = field.get_val(docs[2]);
|
||||
let val4 = field.get_val(docs[3]);
|
||||
let val1 = field.get_val(docs[0] as u64);
|
||||
let val2 = field.get_val(docs[1] as u64);
|
||||
let val3 = field.get_val(docs[2] as u64);
|
||||
let val4 = field.get_val(docs[3] as u64);
|
||||
let val1 = f64_from_fastfield_u64(val1, &self.field_type);
|
||||
let val2 = f64_from_fastfield_u64(val2, &self.field_type);
|
||||
let val3 = f64_from_fastfield_u64(val3, &self.field_type);
|
||||
@@ -180,7 +180,7 @@ impl SegmentStatsCollector {
|
||||
self.stats.collect(val4);
|
||||
}
|
||||
for &doc in iter.remainder() {
|
||||
let val = field.get_val(doc);
|
||||
let val = field.get_val(doc as u64);
|
||||
let val = f64_from_fastfield_u64(val, &self.field_type);
|
||||
self.stats.collect(val);
|
||||
}
|
||||
|
||||
@@ -10,19 +10,21 @@
|
||||
//!
|
||||
//! There are two categories: [Metrics](metric) and [Buckets](bucket).
|
||||
//!
|
||||
//! ## Prerequisite
|
||||
//! Currently aggregations work only on [fast fields](`crate::fastfield`). Single value fast fields
|
||||
//! of type `u64`, `f64`, `i64` and fast fields on text fields.
|
||||
//! # Usage
|
||||
//!
|
||||
//!
|
||||
//! ## Usage
|
||||
//! To use aggregations, build an aggregation request by constructing
|
||||
//! [`Aggregations`](agg_req::Aggregations).
|
||||
//! Create an [`AggregationCollector`] from this request. `AggregationCollector` implements the
|
||||
//! [`Collector`](crate::collector::Collector) trait and can be passed as collector into
|
||||
//! [`Searcher::search()`](crate::Searcher::search).
|
||||
//!
|
||||
//! #### Limitations
|
||||
//!
|
||||
//! ## JSON Format
|
||||
//! Currently aggregations work only on single value fast fields of type `u64`, `f64`, `i64` and
|
||||
//! fast fields on text fields.
|
||||
//!
|
||||
//! # JSON Format
|
||||
//! Aggregations request and result structures de/serialize into elasticsearch compatible JSON.
|
||||
//!
|
||||
//! ```verbatim
|
||||
@@ -33,7 +35,7 @@
|
||||
//! let json_response_string: String = &serde_json::to_string(&agg_res)?;
|
||||
//! ```
|
||||
//!
|
||||
//! ## Supported Aggregations
|
||||
//! # Supported Aggregations
|
||||
//! - [Bucket](bucket)
|
||||
//! - [Histogram](bucket::HistogramAggregation)
|
||||
//! - [Range](bucket::RangeAggregation)
|
||||
|
||||
@@ -177,7 +177,7 @@ where
|
||||
type Fruit = TSegmentCollector::Fruit;
|
||||
|
||||
fn collect(&mut self, doc: u32, score: Score) {
|
||||
let value = self.fast_field_reader.get_val(doc);
|
||||
let value = self.fast_field_reader.get_val(doc as u64);
|
||||
if (self.predicate)(value) {
|
||||
self.segment_collector.collect(doc, score)
|
||||
}
|
||||
|
||||
@@ -94,7 +94,7 @@ impl SegmentCollector for SegmentHistogramCollector {
|
||||
type Fruit = Vec<u64>;
|
||||
|
||||
fn collect(&mut self, doc: DocId, _score: Score) {
|
||||
let value = self.ff_reader.get_val(doc);
|
||||
let value = self.ff_reader.get_val(doc as u64);
|
||||
self.histogram_computer.add_value(value);
|
||||
}
|
||||
|
||||
|
||||
@@ -201,7 +201,7 @@ impl SegmentCollector for FastFieldSegmentCollector {
|
||||
type Fruit = Vec<u64>;
|
||||
|
||||
fn collect(&mut self, doc: DocId, _score: Score) {
|
||||
let val = self.reader.get_val(doc);
|
||||
let val = self.reader.get_val(doc as u64);
|
||||
self.vals.push(val);
|
||||
}
|
||||
|
||||
|
||||
@@ -137,7 +137,7 @@ struct ScorerByFastFieldReader {
|
||||
|
||||
impl CustomSegmentScorer<u64> for ScorerByFastFieldReader {
|
||||
fn score(&mut self, doc: DocId) -> u64 {
|
||||
self.ff_reader.get_val(doc)
|
||||
self.ff_reader.get_val(doc as u64)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -458,7 +458,7 @@ impl TopDocs {
|
||||
///
|
||||
/// // We can now define our actual scoring function
|
||||
/// move |doc: DocId, original_score: Score| {
|
||||
/// let popularity: u64 = popularity_reader.get_val(doc);
|
||||
/// let popularity: u64 = popularity_reader.get_val(doc as u64);
|
||||
/// // Well.. For the sake of the example we use a simple logarithm
|
||||
/// // function.
|
||||
/// let popularity_boost_score = ((2u64 + popularity) as Score).log2();
|
||||
@@ -567,8 +567,8 @@ impl TopDocs {
|
||||
///
|
||||
/// // We can now define our actual scoring function
|
||||
/// move |doc: DocId| {
|
||||
/// let popularity: u64 = popularity_reader.get_val(doc);
|
||||
/// let boosted: u64 = boosted_reader.get_val(doc);
|
||||
/// let popularity: u64 = popularity_reader.get_val(doc as u64);
|
||||
/// let boosted: u64 = boosted_reader.get_val(doc as u64);
|
||||
/// // Score do not have to be `f64` in tantivy.
|
||||
/// // Here we return a couple to get lexicographical order
|
||||
/// // for free.
|
||||
|
||||
@@ -15,11 +15,12 @@ use crate::termdict::TermDictionary;
|
||||
///
|
||||
/// It is safe to delete the segment associated with
|
||||
/// an `InvertedIndexReader`. As long as it is open,
|
||||
/// the [`FileSlice`] it is relying on should
|
||||
/// the `FileSlice` it is relying on should
|
||||
/// stay available.
|
||||
///
|
||||
///
|
||||
/// `InvertedIndexReader` are created by calling
|
||||
/// [`SegmentReader::inverted_index()`](crate::SegmentReader::inverted_index).
|
||||
/// the `SegmentReader`'s [`.inverted_index(...)`] method
|
||||
pub struct InvertedIndexReader {
|
||||
termdict: TermDictionary,
|
||||
postings_file_slice: FileSlice,
|
||||
@@ -74,7 +75,7 @@ impl InvertedIndexReader {
|
||||
///
|
||||
/// This is useful for enumerating through a list of terms,
|
||||
/// and consuming the associated posting lists while avoiding
|
||||
/// reallocating a [`BlockSegmentPostings`].
|
||||
/// reallocating a `BlockSegmentPostings`.
|
||||
///
|
||||
/// # Warning
|
||||
///
|
||||
@@ -95,7 +96,7 @@ impl InvertedIndexReader {
|
||||
/// Returns a block postings given a `Term`.
|
||||
/// This method is for an advanced usage only.
|
||||
///
|
||||
/// Most users should prefer using [`Self::read_postings()`] instead.
|
||||
/// Most user should prefer using `read_postings` instead.
|
||||
pub fn read_block_postings(
|
||||
&self,
|
||||
term: &Term,
|
||||
@@ -109,7 +110,7 @@ impl InvertedIndexReader {
|
||||
/// Returns a block postings given a `term_info`.
|
||||
/// This method is for an advanced usage only.
|
||||
///
|
||||
/// Most users should prefer using [`Self::read_postings()`] instead.
|
||||
/// Most user should prefer using `read_postings` instead.
|
||||
pub fn read_block_postings_from_terminfo(
|
||||
&self,
|
||||
term_info: &TermInfo,
|
||||
@@ -129,7 +130,7 @@ impl InvertedIndexReader {
|
||||
/// Returns a posting object given a `term_info`.
|
||||
/// This method is for an advanced usage only.
|
||||
///
|
||||
/// Most users should prefer using [`Self::read_postings()`] instead.
|
||||
/// Most user should prefer using `read_postings` instead.
|
||||
pub fn read_postings_from_terminfo(
|
||||
&self,
|
||||
term_info: &TermInfo,
|
||||
@@ -163,12 +164,12 @@ impl InvertedIndexReader {
|
||||
/// or `None` if the term has never been encountered and indexed.
|
||||
///
|
||||
/// If the field was not indexed with the indexing options that cover
|
||||
/// the requested options, the returned [`SegmentPostings`] the method does not fail
|
||||
/// the requested options, the returned `SegmentPostings` the method does not fail
|
||||
/// and returns a `SegmentPostings` with as much information as possible.
|
||||
///
|
||||
/// For instance, requesting [`IndexRecordOption::WithFreqs`] for a
|
||||
/// [`TextOptions`](crate::schema::TextOptions) that does not index position
|
||||
/// will return a [`SegmentPostings`] with `DocId`s and frequencies.
|
||||
/// For instance, requesting `IndexRecordOption::Freq` for a
|
||||
/// `TextIndexingOptions` that does not index position will return a `SegmentPostings`
|
||||
/// with `DocId`s and frequencies.
|
||||
pub fn read_postings(
|
||||
&self,
|
||||
term: &Term,
|
||||
@@ -210,7 +211,7 @@ impl InvertedIndexReader {
|
||||
/// Returns a block postings given a `Term`.
|
||||
/// This method is for an advanced usage only.
|
||||
///
|
||||
/// Most users should prefer using [`Self::read_postings()`] instead.
|
||||
/// Most user should prefer using `read_postings` instead.
|
||||
pub async fn warm_postings(
|
||||
&self,
|
||||
term: &Term,
|
||||
|
||||
@@ -216,10 +216,10 @@ impl SegmentReader {
|
||||
/// term dictionary associated with a specific field,
|
||||
/// and opening the posting list associated with any term.
|
||||
///
|
||||
/// If the field is not marked as index, a warning is logged and an empty `InvertedIndexReader`
|
||||
/// If the field is not marked as index, a warn is logged and an empty `InvertedIndexReader`
|
||||
/// is returned.
|
||||
/// Similarly, if the field is marked as indexed but no term has been indexed for the given
|
||||
/// index, an empty `InvertedIndexReader` is returned (but no warning is logged).
|
||||
/// Similarly if the field is marked as indexed but no term has been indexed for the given
|
||||
/// index. an empty `InvertedIndexReader` is returned (but no warning is logged).
|
||||
pub fn inverted_index(&self, field: Field) -> crate::Result<Arc<InvertedIndexReader>> {
|
||||
if let Some(inv_idx_reader) = self
|
||||
.inv_idx_reader_cache
|
||||
|
||||
@@ -13,8 +13,8 @@ use crate::directory::OwnedBytes;
|
||||
/// By contract, whatever happens to the directory file, as long as a FileHandle
|
||||
/// is alive, the data associated with it cannot be altered or destroyed.
|
||||
///
|
||||
/// The underlying behavior is therefore specific to the [`Directory`](crate::Directory) that
|
||||
/// created it. Despite its name, a [`FileSlice`] may or may not directly map to an actual file
|
||||
/// The underlying behavior is therefore specific to the `Directory` that created it.
|
||||
/// Despite its name, a `FileSlice` may or may not directly map to an actual file
|
||||
/// on the filesystem.
|
||||
|
||||
#[async_trait]
|
||||
|
||||
@@ -571,21 +571,9 @@ mod tests {
|
||||
assert_eq!(mmap_directory.get_cache_info().mmapped.len(), 0);
|
||||
}
|
||||
|
||||
fn assert_eventually<P: Fn() -> Option<String>>(predicate: P) {
|
||||
for _ in 0..30 {
|
||||
if predicate().is_none() {
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(200));
|
||||
}
|
||||
if let Some(error_msg) = predicate() {
|
||||
panic!("{}", error_msg);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_mmap_released() {
|
||||
let mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
|
||||
fn test_mmap_released() -> crate::Result<()> {
|
||||
let mmap_directory = MmapDirectory::create_from_tempdir()?;
|
||||
let mut schema_builder: SchemaBuilder = Schema::builder();
|
||||
let text_field = schema_builder.add_text_field("text", TEXT);
|
||||
let schema = schema_builder.build();
|
||||
@@ -594,56 +582,47 @@ mod tests {
|
||||
let index =
|
||||
Index::create(mmap_directory.clone(), schema, IndexSettings::default()).unwrap();
|
||||
|
||||
let mut index_writer = index.writer_for_tests().unwrap();
|
||||
let mut index_writer = index.writer_for_tests()?;
|
||||
let mut log_merge_policy = LogMergePolicy::default();
|
||||
log_merge_policy.set_min_num_segments(3);
|
||||
index_writer.set_merge_policy(Box::new(log_merge_policy));
|
||||
for _num_commits in 0..10 {
|
||||
for _ in 0..10 {
|
||||
index_writer.add_document(doc!(text_field=>"abc")).unwrap();
|
||||
index_writer.add_document(doc!(text_field=>"abc"))?;
|
||||
}
|
||||
index_writer.commit().unwrap();
|
||||
index_writer.commit()?;
|
||||
}
|
||||
|
||||
let reader = index
|
||||
.reader_builder()
|
||||
.reload_policy(ReloadPolicy::Manual)
|
||||
.try_into()
|
||||
.unwrap();
|
||||
.try_into()?;
|
||||
|
||||
for _ in 0..4 {
|
||||
index_writer.add_document(doc!(text_field=>"abc")).unwrap();
|
||||
index_writer.commit().unwrap();
|
||||
reader.reload().unwrap();
|
||||
index_writer.add_document(doc!(text_field=>"abc"))?;
|
||||
index_writer.commit()?;
|
||||
reader.reload()?;
|
||||
}
|
||||
index_writer.wait_merging_threads().unwrap();
|
||||
index_writer.wait_merging_threads()?;
|
||||
|
||||
reader.reload().unwrap();
|
||||
reader.reload()?;
|
||||
let num_segments = reader.searcher().segment_readers().len();
|
||||
assert!(num_segments <= 4);
|
||||
let num_components_except_deletes_and_tempstore =
|
||||
crate::core::SegmentComponent::iterator().len() - 2;
|
||||
let max_num_mmapped = num_components_except_deletes_and_tempstore * num_segments;
|
||||
assert_eventually(|| {
|
||||
let num_mmapped = mmap_directory.get_cache_info().mmapped.len();
|
||||
if num_mmapped > max_num_mmapped {
|
||||
Some(format!(
|
||||
"Expected at most {max_num_mmapped} mmapped files, got {num_mmapped}"
|
||||
))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
assert_eq!(
|
||||
num_segments * num_components_except_deletes_and_tempstore,
|
||||
mmap_directory.get_cache_info().mmapped.len()
|
||||
);
|
||||
}
|
||||
// This test failed on CI. The last Mmap is dropped from the merging thread so there might
|
||||
// be a race condition indeed.
|
||||
assert_eventually(|| {
|
||||
let num_mmapped = mmap_directory.get_cache_info().mmapped.len();
|
||||
if num_mmapped > 0 {
|
||||
Some(format!("Expected no mmapped files, got {num_mmapped}"))
|
||||
} else {
|
||||
None
|
||||
for _ in 0..10 {
|
||||
if mmap_directory.get_cache_info().mmapped.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
});
|
||||
std::thread::sleep(Duration::from_millis(200));
|
||||
}
|
||||
panic!("The cache still contains information. One of the Mmap has not been dropped.");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,9 +32,10 @@ impl BytesFastFieldReader {
|
||||
Ok(BytesFastFieldReader { idx_reader, values })
|
||||
}
|
||||
|
||||
fn range(&self, doc: DocId) -> Range<u32> {
|
||||
let start = self.idx_reader.get_val(doc) as u32;
|
||||
let end = self.idx_reader.get_val(doc + 1) as u32;
|
||||
fn range(&self, doc: DocId) -> Range<u64> {
|
||||
let idx = doc as u64;
|
||||
let start = self.idx_reader.get_val(idx);
|
||||
let end = self.idx_reader.get_val(idx + 1);
|
||||
start..end
|
||||
}
|
||||
|
||||
@@ -47,7 +48,7 @@ impl BytesFastFieldReader {
|
||||
/// Returns the length of the bytes associated with the given `doc`
|
||||
pub fn num_bytes(&self, doc: DocId) -> u64 {
|
||||
let range = self.range(doc);
|
||||
(range.end - range.start) as u64
|
||||
range.end - range.start
|
||||
}
|
||||
|
||||
/// Returns the overall number of bytes in this bytes fast field.
|
||||
@@ -57,7 +58,7 @@ impl BytesFastFieldReader {
|
||||
}
|
||||
|
||||
impl MultiValueLength for BytesFastFieldReader {
|
||||
fn get_range(&self, doc_id: DocId) -> std::ops::Range<u32> {
|
||||
fn get_range(&self, doc_id: DocId) -> std::ops::Range<u64> {
|
||||
self.range(doc_id)
|
||||
}
|
||||
fn get_len(&self, doc_id: DocId) -> u64 {
|
||||
|
||||
@@ -57,15 +57,14 @@ impl BytesFastFieldWriter {
|
||||
|
||||
/// Shift to the next document and add all of the
|
||||
/// matching field values present in the document.
|
||||
pub fn add_document(&mut self, doc: &Document) -> crate::Result<()> {
|
||||
pub fn add_document(&mut self, doc: &Document) {
|
||||
self.next_doc();
|
||||
for field_value in doc.get_all(self.field) {
|
||||
if let Value::Bytes(ref bytes) = field_value {
|
||||
self.vals.extend_from_slice(bytes);
|
||||
return Ok(());
|
||||
return;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Register the bytes associated with a document.
|
||||
|
||||
@@ -7,15 +7,16 @@
|
||||
//! It is designed for the fast random access of some document
|
||||
//! fields given a document id.
|
||||
//!
|
||||
//! Fast fields are useful when a field is required for all or most of
|
||||
//! the `DocSet`: for instance for scoring, grouping, aggregation, filtering, or faceting.
|
||||
//! `FastField` are useful when a field is required for all or most of
|
||||
//! the `DocSet` : for instance for scoring, grouping, filtering, or faceting.
|
||||
//!
|
||||
//!
|
||||
//! Fields have to be declared as `FAST` in the schema.
|
||||
//! Currently supported fields are: u64, i64, f64, bytes and text.
|
||||
//! Fields have to be declared as `FAST` in the schema.
|
||||
//! Currently supported fields are: u64, i64, f64 and bytes.
|
||||
//!
|
||||
//! Fast fields are stored in with [different codecs](fastfield_codecs). The best codec is detected
|
||||
//! automatically, when serializing.
|
||||
//! u64, i64 and f64 fields are stored in a bit-packed fashion so that
|
||||
//! their memory usage is directly linear with the amplitude of the
|
||||
//! values stored.
|
||||
//!
|
||||
//! Read access performance is comparable to that of an array lookup.
|
||||
|
||||
@@ -25,15 +26,11 @@ pub use self::alive_bitset::{intersect_alive_bitsets, write_alive_bitset, AliveB
|
||||
pub use self::bytes::{BytesFastFieldReader, BytesFastFieldWriter};
|
||||
pub use self::error::{FastFieldNotAvailableError, Result};
|
||||
pub use self::facet_reader::FacetReader;
|
||||
pub(crate) use self::multivalued::{get_fastfield_codecs_for_multivalue, MultivalueStartIndex};
|
||||
pub use self::multivalued::{
|
||||
MultiValueU128FastFieldWriter, MultiValuedFastFieldReader, MultiValuedFastFieldWriter,
|
||||
MultiValuedU128FastFieldReader,
|
||||
};
|
||||
pub(crate) use self::multivalued::MultivalueStartIndex;
|
||||
pub use self::multivalued::{MultiValuedFastFieldReader, MultiValuedFastFieldWriter};
|
||||
pub use self::readers::FastFieldReaders;
|
||||
pub(crate) use self::readers::{type_and_cardinality, FastType};
|
||||
pub use self::serializer::{Column, CompositeFastFieldSerializer};
|
||||
use self::writer::unexpected_value;
|
||||
pub use self::writer::{FastFieldsWriter, IntFastFieldWriter};
|
||||
use crate::schema::{Type, Value};
|
||||
use crate::{DateTime, DocId};
|
||||
@@ -51,7 +48,7 @@ mod writer;
|
||||
/// for a doc_id
|
||||
pub trait MultiValueLength {
|
||||
/// returns the positions for a docid
|
||||
fn get_range(&self, doc_id: DocId) -> std::ops::Range<u32>;
|
||||
fn get_range(&self, doc_id: DocId) -> std::ops::Range<u64>;
|
||||
/// returns the num of values associated with a doc_id
|
||||
fn get_len(&self, doc_id: DocId) -> u64;
|
||||
/// returns the sum of num values for all doc_ids
|
||||
@@ -120,16 +117,15 @@ impl FastValue for DateTime {
|
||||
}
|
||||
}
|
||||
|
||||
fn value_to_u64(value: &Value) -> crate::Result<u64> {
|
||||
let value = match value {
|
||||
fn value_to_u64(value: &Value) -> u64 {
|
||||
match value {
|
||||
Value::U64(val) => val.to_u64(),
|
||||
Value::I64(val) => val.to_u64(),
|
||||
Value::F64(val) => val.to_u64(),
|
||||
Value::Bool(val) => val.to_u64(),
|
||||
Value::Date(val) => val.to_u64(),
|
||||
_ => return Err(unexpected_value("u64/i64/f64/bool/date", value)),
|
||||
};
|
||||
Ok(value)
|
||||
_ => panic!("Expected a u64/i64/f64/bool/date field, got {:?} ", value),
|
||||
}
|
||||
}
|
||||
|
||||
/// The fast field type
|
||||
@@ -184,9 +180,9 @@ mod tests {
|
||||
#[test]
|
||||
pub fn test_fastfield() {
|
||||
let test_fastfield = fastfield_codecs::serialize_and_load(&[100u64, 200u64, 300u64][..]);
|
||||
assert_eq!(test_fastfield.get_val(0), 100);
|
||||
assert_eq!(test_fastfield.get_val(1), 200);
|
||||
assert_eq!(test_fastfield.get_val(2), 300);
|
||||
assert_eq!(test_fastfield.get_val(0u64), 100);
|
||||
assert_eq!(test_fastfield.get_val(1u64), 200);
|
||||
assert_eq!(test_fastfield.get_val(2u64), 300);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -203,15 +199,9 @@ mod tests {
|
||||
let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
|
||||
let mut serializer = CompositeFastFieldSerializer::from_write(write).unwrap();
|
||||
let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA);
|
||||
fast_field_writers
|
||||
.add_document(&doc!(*FIELD=>13u64))
|
||||
.unwrap();
|
||||
fast_field_writers
|
||||
.add_document(&doc!(*FIELD=>14u64))
|
||||
.unwrap();
|
||||
fast_field_writers
|
||||
.add_document(&doc!(*FIELD=>2u64))
|
||||
.unwrap();
|
||||
fast_field_writers.add_document(&doc!(*FIELD=>13u64));
|
||||
fast_field_writers.add_document(&doc!(*FIELD=>14u64));
|
||||
fast_field_writers.add_document(&doc!(*FIELD=>2u64));
|
||||
fast_field_writers
|
||||
.serialize(&mut serializer, &HashMap::new(), None)
|
||||
.unwrap();
|
||||
@@ -236,33 +226,15 @@ mod tests {
|
||||
let write: WritePtr = directory.open_write(Path::new("test"))?;
|
||||
let mut serializer = CompositeFastFieldSerializer::from_write(write)?;
|
||||
let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA);
|
||||
fast_field_writers
|
||||
.add_document(&doc!(*FIELD=>4u64))
|
||||
.unwrap();
|
||||
fast_field_writers
|
||||
.add_document(&doc!(*FIELD=>14_082_001u64))
|
||||
.unwrap();
|
||||
fast_field_writers
|
||||
.add_document(&doc!(*FIELD=>3_052u64))
|
||||
.unwrap();
|
||||
fast_field_writers
|
||||
.add_document(&doc!(*FIELD=>9_002u64))
|
||||
.unwrap();
|
||||
fast_field_writers
|
||||
.add_document(&doc!(*FIELD=>15_001u64))
|
||||
.unwrap();
|
||||
fast_field_writers
|
||||
.add_document(&doc!(*FIELD=>777u64))
|
||||
.unwrap();
|
||||
fast_field_writers
|
||||
.add_document(&doc!(*FIELD=>1_002u64))
|
||||
.unwrap();
|
||||
fast_field_writers
|
||||
.add_document(&doc!(*FIELD=>1_501u64))
|
||||
.unwrap();
|
||||
fast_field_writers
|
||||
.add_document(&doc!(*FIELD=>215u64))
|
||||
.unwrap();
|
||||
fast_field_writers.add_document(&doc!(*FIELD=>4u64));
|
||||
fast_field_writers.add_document(&doc!(*FIELD=>14_082_001u64));
|
||||
fast_field_writers.add_document(&doc!(*FIELD=>3_052u64));
|
||||
fast_field_writers.add_document(&doc!(*FIELD=>9_002u64));
|
||||
fast_field_writers.add_document(&doc!(*FIELD=>15_001u64));
|
||||
fast_field_writers.add_document(&doc!(*FIELD=>777u64));
|
||||
fast_field_writers.add_document(&doc!(*FIELD=>1_002u64));
|
||||
fast_field_writers.add_document(&doc!(*FIELD=>1_501u64));
|
||||
fast_field_writers.add_document(&doc!(*FIELD=>215u64));
|
||||
fast_field_writers.serialize(&mut serializer, &HashMap::new(), None)?;
|
||||
serializer.close()?;
|
||||
}
|
||||
@@ -298,9 +270,7 @@ mod tests {
|
||||
let mut serializer = CompositeFastFieldSerializer::from_write(write).unwrap();
|
||||
let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA);
|
||||
for _ in 0..10_000 {
|
||||
fast_field_writers
|
||||
.add_document(&doc!(*FIELD=>100_000u64))
|
||||
.unwrap();
|
||||
fast_field_writers.add_document(&doc!(*FIELD=>100_000u64));
|
||||
}
|
||||
fast_field_writers
|
||||
.serialize(&mut serializer, &HashMap::new(), None)
|
||||
@@ -333,13 +303,9 @@ mod tests {
|
||||
let mut serializer = CompositeFastFieldSerializer::from_write(write).unwrap();
|
||||
let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA);
|
||||
// forcing the amplitude to be high
|
||||
fast_field_writers
|
||||
.add_document(&doc!(*FIELD=>0u64))
|
||||
.unwrap();
|
||||
fast_field_writers.add_document(&doc!(*FIELD=>0u64));
|
||||
for i in 0u64..10_000u64 {
|
||||
fast_field_writers
|
||||
.add_document(&doc!(*FIELD=>5_000_000_000_000_000_000u64 + i))
|
||||
.unwrap();
|
||||
fast_field_writers.add_document(&doc!(*FIELD=>5_000_000_000_000_000_000u64 + i));
|
||||
}
|
||||
fast_field_writers
|
||||
.serialize(&mut serializer, &HashMap::new(), None)
|
||||
@@ -381,7 +347,7 @@ mod tests {
|
||||
for i in -100i64..10_000i64 {
|
||||
let mut doc = Document::default();
|
||||
doc.add_i64(i64_field, i);
|
||||
fast_field_writers.add_document(&doc).unwrap();
|
||||
fast_field_writers.add_document(&doc);
|
||||
}
|
||||
fast_field_writers
|
||||
.serialize(&mut serializer, &HashMap::new(), None)
|
||||
@@ -402,7 +368,7 @@ mod tests {
|
||||
assert_eq!(fast_field_reader.min_value(), -100i64);
|
||||
assert_eq!(fast_field_reader.max_value(), 9_999i64);
|
||||
for (doc, i) in (-100i64..10_000i64).enumerate() {
|
||||
assert_eq!(fast_field_reader.get_val(doc as u32), i);
|
||||
assert_eq!(fast_field_reader.get_val(doc as u64), i);
|
||||
}
|
||||
let mut buffer = vec![0i64; 100];
|
||||
fast_field_reader.get_range(53, &mut buffer[..]);
|
||||
@@ -426,7 +392,7 @@ mod tests {
|
||||
let mut serializer = CompositeFastFieldSerializer::from_write(write).unwrap();
|
||||
let mut fast_field_writers = FastFieldsWriter::from_schema(&schema);
|
||||
let doc = Document::default();
|
||||
fast_field_writers.add_document(&doc).unwrap();
|
||||
fast_field_writers.add_document(&doc);
|
||||
fast_field_writers
|
||||
.serialize(&mut serializer, &HashMap::new(), None)
|
||||
.unwrap();
|
||||
@@ -469,7 +435,7 @@ mod tests {
|
||||
let mut serializer = CompositeFastFieldSerializer::from_write(write)?;
|
||||
let mut fast_field_writers = FastFieldsWriter::from_schema(&SCHEMA);
|
||||
for &x in &permutation {
|
||||
fast_field_writers.add_document(&doc!(*FIELD=>x)).unwrap();
|
||||
fast_field_writers.add_document(&doc!(*FIELD=>x));
|
||||
}
|
||||
fast_field_writers.serialize(&mut serializer, &HashMap::new(), None)?;
|
||||
serializer.close()?;
|
||||
@@ -484,7 +450,7 @@ mod tests {
|
||||
let fast_field_reader = open::<u64>(data)?;
|
||||
|
||||
for a in 0..n {
|
||||
assert_eq!(fast_field_reader.get_val(a as u32), permutation[a as usize]);
|
||||
assert_eq!(fast_field_reader.get_val(a as u64), permutation[a as usize]);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -819,14 +785,10 @@ mod tests {
|
||||
let write: WritePtr = directory.open_write(path).unwrap();
|
||||
let mut serializer = CompositeFastFieldSerializer::from_write(write).unwrap();
|
||||
let mut fast_field_writers = FastFieldsWriter::from_schema(&schema);
|
||||
fast_field_writers.add_document(&doc!(field=>true)).unwrap();
|
||||
fast_field_writers
|
||||
.add_document(&doc!(field=>false))
|
||||
.unwrap();
|
||||
fast_field_writers.add_document(&doc!(field=>true)).unwrap();
|
||||
fast_field_writers
|
||||
.add_document(&doc!(field=>false))
|
||||
.unwrap();
|
||||
fast_field_writers.add_document(&doc!(field=>true));
|
||||
fast_field_writers.add_document(&doc!(field=>false));
|
||||
fast_field_writers.add_document(&doc!(field=>true));
|
||||
fast_field_writers.add_document(&doc!(field=>false));
|
||||
fast_field_writers
|
||||
.serialize(&mut serializer, &HashMap::new(), None)
|
||||
.unwrap();
|
||||
@@ -860,10 +822,8 @@ mod tests {
|
||||
let mut serializer = CompositeFastFieldSerializer::from_write(write).unwrap();
|
||||
let mut fast_field_writers = FastFieldsWriter::from_schema(&schema);
|
||||
for _ in 0..50 {
|
||||
fast_field_writers.add_document(&doc!(field=>true)).unwrap();
|
||||
fast_field_writers
|
||||
.add_document(&doc!(field=>false))
|
||||
.unwrap();
|
||||
fast_field_writers.add_document(&doc!(field=>true));
|
||||
fast_field_writers.add_document(&doc!(field=>false));
|
||||
}
|
||||
fast_field_writers
|
||||
.serialize(&mut serializer, &HashMap::new(), None)
|
||||
@@ -897,7 +857,7 @@ mod tests {
|
||||
let mut serializer = CompositeFastFieldSerializer::from_write(write)?;
|
||||
let mut fast_field_writers = FastFieldsWriter::from_schema(&schema);
|
||||
let doc = Document::default();
|
||||
fast_field_writers.add_document(&doc).unwrap();
|
||||
fast_field_writers.add_document(&doc);
|
||||
fast_field_writers.serialize(&mut serializer, &HashMap::new(), None)?;
|
||||
serializer.close()?;
|
||||
}
|
||||
@@ -923,7 +883,7 @@ mod tests {
|
||||
CompositeFastFieldSerializer::from_write_with_codec(write, codec_types).unwrap();
|
||||
let mut fast_field_writers = FastFieldsWriter::from_schema(schema);
|
||||
for doc in docs {
|
||||
fast_field_writers.add_document(doc).unwrap();
|
||||
fast_field_writers.add_document(doc);
|
||||
}
|
||||
fast_field_writers
|
||||
.serialize(&mut serializer, &HashMap::new(), None)
|
||||
@@ -976,7 +936,7 @@ mod tests {
|
||||
let test_fastfield = open::<DateTime>(file.read_bytes()?)?;
|
||||
|
||||
for (i, time) in times.iter().enumerate() {
|
||||
assert_eq!(test_fastfield.get_val(i as u32), time.truncate(precision));
|
||||
assert_eq!(test_fastfield.get_val(i as u64), time.truncate(precision));
|
||||
}
|
||||
Ok(len)
|
||||
}
|
||||
|
||||
@@ -1,21 +1,9 @@
|
||||
mod reader;
|
||||
mod writer;
|
||||
|
||||
use fastfield_codecs::FastFieldCodecType;
|
||||
|
||||
pub use self::reader::{MultiValuedFastFieldReader, MultiValuedU128FastFieldReader};
|
||||
pub use self::reader::MultiValuedFastFieldReader;
|
||||
pub use self::writer::MultiValuedFastFieldWriter;
|
||||
pub(crate) use self::writer::MultivalueStartIndex;
|
||||
pub use self::writer::{MultiValueU128FastFieldWriter, MultiValuedFastFieldWriter};
|
||||
|
||||
/// The valid codecs for multivalue values excludes the linear interpolation codec.
|
||||
///
|
||||
/// This limitation is only valid for the values, not the offset index of the multivalue index.
|
||||
pub(crate) fn get_fastfield_codecs_for_multivalue() -> [FastFieldCodecType; 2] {
|
||||
[
|
||||
FastFieldCodecType::Bitpacked,
|
||||
FastFieldCodecType::BlockwiseLinear,
|
||||
]
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
@@ -515,7 +503,7 @@ mod bench {
|
||||
for val in block {
|
||||
doc.add_u64(field, *val);
|
||||
}
|
||||
fast_field_writers.add_document(&doc).unwrap();
|
||||
fast_field_writers.add_document(&doc);
|
||||
}
|
||||
fast_field_writers
|
||||
.serialize(&mut serializer, &HashMap::new(), None)
|
||||
@@ -573,7 +561,7 @@ mod bench {
|
||||
for val in block {
|
||||
doc.add_u64(field, *val);
|
||||
}
|
||||
fast_field_writers.add_document(&doc).unwrap();
|
||||
fast_field_writers.add_document(&doc);
|
||||
}
|
||||
fast_field_writers
|
||||
.serialize(&mut serializer, &HashMap::new(), None)
|
||||
@@ -606,7 +594,7 @@ mod bench {
|
||||
for val in block {
|
||||
doc.add_u64(field, *val);
|
||||
}
|
||||
fast_field_writers.add_document(&doc).unwrap();
|
||||
fast_field_writers.add_document(&doc);
|
||||
}
|
||||
fast_field_writers
|
||||
.serialize(&mut serializer, &HashMap::new(), Some(&doc_id_mapping))
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use std::ops::{Range, RangeInclusive};
|
||||
use std::ops::Range;
|
||||
use std::sync::Arc;
|
||||
|
||||
use fastfield_codecs::{Column, MonotonicallyMappableToU128};
|
||||
use fastfield_codecs::Column;
|
||||
|
||||
use crate::fastfield::{FastValue, MultiValueLength};
|
||||
use crate::DocId;
|
||||
@@ -33,19 +33,19 @@ impl<Item: FastValue> MultiValuedFastFieldReader<Item> {
|
||||
/// Returns `[start, end)`, such that the values associated with
|
||||
/// the given document are `start..end`.
|
||||
#[inline]
|
||||
fn range(&self, doc: DocId) -> Range<u32> {
|
||||
let start = self.idx_reader.get_val(doc) as u32;
|
||||
let end = self.idx_reader.get_val(doc + 1) as u32;
|
||||
fn range(&self, doc: DocId) -> Range<u64> {
|
||||
let idx = doc as u64;
|
||||
let start = self.idx_reader.get_val(idx);
|
||||
let end = self.idx_reader.get_val(idx + 1);
|
||||
start..end
|
||||
}
|
||||
|
||||
/// Returns the array of values associated with the given `doc`.
|
||||
#[inline]
|
||||
fn get_vals_for_range(&self, range: Range<u32>, vals: &mut Vec<Item>) {
|
||||
fn get_vals_for_range(&self, range: Range<u64>, vals: &mut Vec<Item>) {
|
||||
let len = (range.end - range.start) as usize;
|
||||
vals.resize(len, Item::make_zero());
|
||||
self.vals_reader
|
||||
.get_range(range.start as u64, &mut vals[..]);
|
||||
self.vals_reader.get_range(range.start, &mut vals[..]);
|
||||
}
|
||||
|
||||
/// Returns the array of values associated with the given `doc`.
|
||||
@@ -88,7 +88,7 @@ impl<Item: FastValue> MultiValuedFastFieldReader<Item> {
|
||||
}
|
||||
|
||||
impl<Item: FastValue> MultiValueLength for MultiValuedFastFieldReader<Item> {
|
||||
fn get_range(&self, doc_id: DocId) -> Range<u32> {
|
||||
fn get_range(&self, doc_id: DocId) -> Range<u64> {
|
||||
self.range(doc_id)
|
||||
}
|
||||
fn get_len(&self, doc_id: DocId) -> u64 {
|
||||
@@ -99,183 +99,12 @@ impl<Item: FastValue> MultiValueLength for MultiValuedFastFieldReader<Item> {
|
||||
self.total_num_vals() as u64
|
||||
}
|
||||
}
|
||||
|
||||
/// Reader for a multivalued `u128` fast field.
|
||||
///
|
||||
/// The reader is implemented as a `u64` fast field for the index and a `u128` fast field.
|
||||
///
|
||||
/// The `vals_reader` will access the concatenated list of all
|
||||
/// values for all reader.
|
||||
/// The `idx_reader` associated, for each document, the index of its first value.
|
||||
#[derive(Clone)]
|
||||
pub struct MultiValuedU128FastFieldReader<T: MonotonicallyMappableToU128> {
|
||||
idx_reader: Arc<dyn Column<u64>>,
|
||||
vals_reader: Arc<dyn Column<T>>,
|
||||
}
|
||||
|
||||
impl<T: MonotonicallyMappableToU128> MultiValuedU128FastFieldReader<T> {
|
||||
pub(crate) fn open(
|
||||
idx_reader: Arc<dyn Column<u64>>,
|
||||
vals_reader: Arc<dyn Column<T>>,
|
||||
) -> MultiValuedU128FastFieldReader<T> {
|
||||
Self {
|
||||
idx_reader,
|
||||
vals_reader,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `[start, end)`, such that the values associated
|
||||
/// to the given document are `start..end`.
|
||||
#[inline]
|
||||
fn range(&self, doc: DocId) -> Range<u32> {
|
||||
let start = self.idx_reader.get_val(doc) as u32;
|
||||
let end = self.idx_reader.get_val(doc + 1) as u32;
|
||||
start..end
|
||||
}
|
||||
|
||||
/// Returns the array of values associated to the given `doc`.
|
||||
#[inline]
|
||||
pub fn get_first_val(&self, doc: DocId) -> Option<T> {
|
||||
let range = self.range(doc);
|
||||
if range.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(self.vals_reader.get_val(range.start))
|
||||
}
|
||||
|
||||
/// Returns the array of values associated to the given `doc`.
|
||||
#[inline]
|
||||
fn get_vals_for_range(&self, range: Range<u32>, vals: &mut Vec<T>) {
|
||||
let len = (range.end - range.start) as usize;
|
||||
vals.resize(len, T::from_u128(0));
|
||||
self.vals_reader
|
||||
.get_range(range.start as u64, &mut vals[..]);
|
||||
}
|
||||
|
||||
/// Returns the array of values associated to the given `doc`.
|
||||
#[inline]
|
||||
pub fn get_vals(&self, doc: DocId, vals: &mut Vec<T>) {
|
||||
let range = self.range(doc);
|
||||
self.get_vals_for_range(range, vals);
|
||||
}
|
||||
|
||||
/// Returns all docids which are in the provided value range
|
||||
pub fn get_positions_for_value_range(
|
||||
&self,
|
||||
value_range: RangeInclusive<T>,
|
||||
doc_id_range: Range<u32>,
|
||||
) -> Vec<DocId> {
|
||||
let mut positions = Vec::new(); // TODO replace
|
||||
self.vals_reader
|
||||
.get_positions_for_value_range(value_range, doc_id_range, &mut positions);
|
||||
|
||||
positions_to_docids(&positions, self.idx_reader.as_ref())
|
||||
}
|
||||
|
||||
/// Iterates over all elements in the fast field
|
||||
pub fn iter(&self) -> impl Iterator<Item = T> + '_ {
|
||||
self.vals_reader.iter()
|
||||
}
|
||||
|
||||
/// Returns the minimum value for this fast field.
|
||||
///
|
||||
/// The min value does not take in account of possible
|
||||
/// deleted document, and should be considered as a lower bound
|
||||
/// of the actual mimimum value.
|
||||
pub fn min_value(&self) -> T {
|
||||
self.vals_reader.min_value()
|
||||
}
|
||||
|
||||
/// Returns the maximum value for this fast field.
|
||||
///
|
||||
/// The max value does not take in account of possible
|
||||
/// deleted document, and should be considered as an upper bound
|
||||
/// of the actual maximum value.
|
||||
pub fn max_value(&self) -> T {
|
||||
self.vals_reader.max_value()
|
||||
}
|
||||
|
||||
/// Returns the number of values associated with the document `DocId`.
|
||||
#[inline]
|
||||
pub fn num_vals(&self, doc: DocId) -> usize {
|
||||
let range = self.range(doc);
|
||||
(range.end - range.start) as usize
|
||||
}
|
||||
|
||||
/// Returns the overall number of values in this field.
|
||||
#[inline]
|
||||
pub fn total_num_vals(&self) -> u64 {
|
||||
self.idx_reader.max_value()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: MonotonicallyMappableToU128> MultiValueLength for MultiValuedU128FastFieldReader<T> {
|
||||
fn get_range(&self, doc_id: DocId) -> std::ops::Range<u32> {
|
||||
self.range(doc_id)
|
||||
}
|
||||
fn get_len(&self, doc_id: DocId) -> u64 {
|
||||
self.num_vals(doc_id) as u64
|
||||
}
|
||||
fn get_total_len(&self) -> u64 {
|
||||
self.total_num_vals() as u64
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts a list of positions of values in a 1:n index to the corresponding list of DocIds.
|
||||
///
|
||||
/// Since there is no index for value pos -> docid, but docid -> value pos range, we scan the index.
|
||||
///
|
||||
/// Correctness: positions needs to be sorted. idx_reader needs to contain monotonically increasing
|
||||
/// positions.
|
||||
///
|
||||
/// TODO: Instead of a linear scan we can employ a expotential search into binary search to match a
|
||||
/// docid to its value position.
|
||||
fn positions_to_docids<C: Column + ?Sized>(positions: &[u32], idx_reader: &C) -> Vec<DocId> {
|
||||
let mut docs = vec![];
|
||||
let mut cur_doc = 0u32;
|
||||
let mut last_doc = None;
|
||||
|
||||
for pos in positions {
|
||||
loop {
|
||||
let end = idx_reader.get_val(cur_doc + 1) as u32;
|
||||
if end > *pos {
|
||||
// avoid duplicates
|
||||
if Some(cur_doc) == last_doc {
|
||||
break;
|
||||
}
|
||||
docs.push(cur_doc);
|
||||
last_doc = Some(cur_doc);
|
||||
break;
|
||||
}
|
||||
cur_doc += 1;
|
||||
}
|
||||
}
|
||||
|
||||
docs
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
use fastfield_codecs::VecColumn;
|
||||
|
||||
use crate::core::Index;
|
||||
use crate::fastfield::multivalued::reader::positions_to_docids;
|
||||
use crate::schema::{Cardinality, Facet, FacetOptions, NumericOptions, Schema};
|
||||
|
||||
#[test]
|
||||
fn test_positions_to_docid() {
|
||||
let positions = vec![10u32, 11, 15, 20, 21, 22];
|
||||
|
||||
let offsets = vec![0, 10, 12, 15, 22, 23];
|
||||
{
|
||||
let column = VecColumn::from(&offsets);
|
||||
|
||||
let docids = positions_to_docids(&positions, &column);
|
||||
assert_eq!(docids, vec![1, 3, 4]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multifastfield_reader() -> crate::Result<()> {
|
||||
let mut schema_builder = Schema::builder();
|
||||
|
||||
@@ -1,12 +1,8 @@
|
||||
use std::io;
|
||||
|
||||
use fastfield_codecs::{
|
||||
Column, MonotonicallyMappableToU128, MonotonicallyMappableToU64, VecColumn,
|
||||
};
|
||||
use rustc_hash::FxHashMap;
|
||||
use fastfield_codecs::{Column, MonotonicallyMappableToU64, VecColumn};
|
||||
use fnv::FnvHashMap;
|
||||
|
||||
use super::get_fastfield_codecs_for_multivalue;
|
||||
use crate::fastfield::writer::unexpected_value;
|
||||
use crate::fastfield::{value_to_u64, CompositeFastFieldSerializer, FastFieldType};
|
||||
use crate::indexer::doc_id_mapping::DocIdMapping;
|
||||
use crate::postings::UnorderedTermId;
|
||||
@@ -82,11 +78,11 @@ impl MultiValuedFastFieldWriter {
|
||||
|
||||
/// Shift to the next document and adds
|
||||
/// all of the matching field values present in the document.
|
||||
pub fn add_document(&mut self, doc: &Document) -> crate::Result<()> {
|
||||
pub fn add_document(&mut self, doc: &Document) {
|
||||
self.next_doc();
|
||||
// facets/texts are indexed in the `SegmentWriter` as we encode their unordered id.
|
||||
if self.fast_field_type.is_storing_term_ids() {
|
||||
return Ok(());
|
||||
return;
|
||||
}
|
||||
for field_value in doc.field_values() {
|
||||
if field_value.field == self.field {
|
||||
@@ -95,12 +91,11 @@ impl MultiValuedFastFieldWriter {
|
||||
(Some(precision), Value::Date(date_val)) => {
|
||||
date_val.truncate(precision).to_u64()
|
||||
}
|
||||
_ => value_to_u64(value)?,
|
||||
_ => value_to_u64(value),
|
||||
};
|
||||
self.add_val(value_u64);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns an iterator over values per doc_id in ascending doc_id order.
|
||||
@@ -144,7 +139,7 @@ impl MultiValuedFastFieldWriter {
|
||||
pub fn serialize(
|
||||
mut self,
|
||||
serializer: &mut CompositeFastFieldSerializer,
|
||||
term_mapping_opt: Option<&FxHashMap<UnorderedTermId, TermOrdinal>>,
|
||||
term_mapping_opt: Option<&FnvHashMap<UnorderedTermId, TermOrdinal>>,
|
||||
doc_id_map: Option<&DocIdMapping>,
|
||||
) -> io::Result<()> {
|
||||
{
|
||||
@@ -199,12 +194,7 @@ impl MultiValuedFastFieldWriter {
|
||||
}
|
||||
}
|
||||
let col = VecColumn::from(&values[..]);
|
||||
serializer.create_auto_detect_u64_fast_field_with_idx_and_codecs(
|
||||
self.field,
|
||||
col,
|
||||
1,
|
||||
&get_fastfield_codecs_for_multivalue(),
|
||||
)?;
|
||||
serializer.create_auto_detect_u64_fast_field_with_idx(self.field, col, 1)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -219,7 +209,7 @@ pub(crate) struct MultivalueStartIndex<'a, C: Column> {
|
||||
|
||||
impl<'a, C: Column> MultivalueStartIndex<'a, C> {
|
||||
pub fn new(column: &'a C, doc_id_map: &'a DocIdMapping) -> Self {
|
||||
assert_eq!(column.num_vals(), doc_id_map.num_old_doc_ids() as u32 + 1);
|
||||
assert_eq!(column.num_vals(), doc_id_map.num_old_doc_ids() as u64 + 1);
|
||||
let (min, max) =
|
||||
tantivy_bitpacker::minmax(iter_remapped_multivalue_index(doc_id_map, column))
|
||||
.unwrap_or((0u64, 0u64));
|
||||
@@ -232,7 +222,7 @@ impl<'a, C: Column> MultivalueStartIndex<'a, C> {
|
||||
}
|
||||
}
|
||||
impl<'a, C: Column> Column for MultivalueStartIndex<'a, C> {
|
||||
fn get_val(&self, _idx: u32) -> u64 {
|
||||
fn get_val(&self, _idx: u64) -> u64 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
@@ -244,8 +234,8 @@ impl<'a, C: Column> Column for MultivalueStartIndex<'a, C> {
|
||||
self.max
|
||||
}
|
||||
|
||||
fn num_vals(&self) -> u32 {
|
||||
(self.doc_id_map.num_new_doc_ids() + 1) as u32
|
||||
fn num_vals(&self) -> u64 {
|
||||
(self.doc_id_map.num_new_doc_ids() + 1) as u64
|
||||
}
|
||||
|
||||
fn iter(&self) -> Box<dyn Iterator<Item = u64> + '_> {
|
||||
@@ -261,149 +251,15 @@ fn iter_remapped_multivalue_index<'a, C: Column>(
|
||||
column: &'a C,
|
||||
) -> impl Iterator<Item = u64> + 'a {
|
||||
let mut offset = 0;
|
||||
std::iter::once(0).chain(doc_id_map.iter_old_doc_ids().map(move |old_doc| {
|
||||
let num_vals_for_doc = column.get_val(old_doc + 1) - column.get_val(old_doc);
|
||||
offset += num_vals_for_doc;
|
||||
offset as u64
|
||||
}))
|
||||
}
|
||||
|
||||
/// Writer for multi-valued (as in, more than one value per document)
|
||||
/// int fast field.
|
||||
///
|
||||
/// This `Writer` is only useful for advanced users.
|
||||
/// The normal way to get your multivalued int in your index
|
||||
/// is to
|
||||
/// - declare your field with fast set to `Cardinality::MultiValues`
|
||||
/// in your schema
|
||||
/// - add your document simply by calling `.add_document(...)`.
|
||||
///
|
||||
/// The `MultiValuedFastFieldWriter` can be acquired from the
|
||||
|
||||
pub struct MultiValueU128FastFieldWriter {
|
||||
field: Field,
|
||||
vals: Vec<u128>,
|
||||
doc_index: Vec<u64>,
|
||||
}
|
||||
|
||||
impl MultiValueU128FastFieldWriter {
|
||||
/// Creates a new `U128MultiValueFastFieldWriter`
|
||||
pub(crate) fn new(field: Field) -> Self {
|
||||
MultiValueU128FastFieldWriter {
|
||||
field,
|
||||
vals: Vec::new(),
|
||||
doc_index: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The memory used (inclusive childs)
|
||||
pub fn mem_usage(&self) -> usize {
|
||||
self.vals.capacity() * std::mem::size_of::<UnorderedTermId>()
|
||||
+ self.doc_index.capacity() * std::mem::size_of::<u64>()
|
||||
}
|
||||
|
||||
/// Finalize the current document.
|
||||
pub(crate) fn next_doc(&mut self) {
|
||||
self.doc_index.push(self.vals.len() as u64);
|
||||
}
|
||||
|
||||
/// Pushes a new value to the current document.
|
||||
pub(crate) fn add_val(&mut self, val: u128) {
|
||||
self.vals.push(val);
|
||||
}
|
||||
|
||||
/// Shift to the next document and adds
|
||||
/// all of the matching field values present in the document.
|
||||
pub fn add_document(&mut self, doc: &Document) -> crate::Result<()> {
|
||||
self.next_doc();
|
||||
for field_value in doc.field_values() {
|
||||
if field_value.field == self.field {
|
||||
let value = field_value.value();
|
||||
let ip_addr = value
|
||||
.as_ip_addr()
|
||||
.ok_or_else(|| unexpected_value("ip", value))?;
|
||||
let ip_addr_u128 = ip_addr.to_u128();
|
||||
self.add_val(ip_addr_u128);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns an iterator over values per doc_id in ascending doc_id order.
|
||||
///
|
||||
/// Normally the order is simply iterating self.doc_id_index.
|
||||
/// With doc_id_map it accounts for the new mapping, returning values in the order of the
|
||||
/// new doc_ids.
|
||||
fn get_ordered_values<'a: 'b, 'b>(
|
||||
&'a self,
|
||||
doc_id_map: Option<&'b DocIdMapping>,
|
||||
) -> impl Iterator<Item = &'b [u128]> {
|
||||
get_ordered_values(&self.vals, &self.doc_index, doc_id_map)
|
||||
}
|
||||
|
||||
/// Serializes fast field values.
|
||||
pub fn serialize(
|
||||
mut self,
|
||||
serializer: &mut CompositeFastFieldSerializer,
|
||||
doc_id_map: Option<&DocIdMapping>,
|
||||
) -> io::Result<()> {
|
||||
{
|
||||
// writing the offset index
|
||||
//
|
||||
self.doc_index.push(self.vals.len() as u64);
|
||||
let col = VecColumn::from(&self.doc_index[..]);
|
||||
if let Some(doc_id_map) = doc_id_map {
|
||||
let multi_value_start_index = MultivalueStartIndex::new(&col, doc_id_map);
|
||||
serializer.create_auto_detect_u64_fast_field_with_idx(
|
||||
self.field,
|
||||
multi_value_start_index,
|
||||
0,
|
||||
)?;
|
||||
} else {
|
||||
serializer.create_auto_detect_u64_fast_field_with_idx(self.field, col, 0)?;
|
||||
}
|
||||
}
|
||||
{
|
||||
let iter_gen = || self.get_ordered_values(doc_id_map).flatten().cloned();
|
||||
|
||||
serializer.create_u128_fast_field_with_idx(
|
||||
self.field,
|
||||
iter_gen,
|
||||
self.vals.len() as u32,
|
||||
1,
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns an iterator over values per doc_id in ascending doc_id order.
|
||||
///
|
||||
/// Normally the order is simply iterating self.doc_id_index.
|
||||
/// With doc_id_map it accounts for the new mapping, returning values in the order of the
|
||||
/// new doc_ids.
|
||||
fn get_ordered_values<'a: 'b, 'b, T>(
|
||||
vals: &'a [T],
|
||||
doc_index: &'a [u64],
|
||||
doc_id_map: Option<&'b DocIdMapping>,
|
||||
) -> impl Iterator<Item = &'b [T]> {
|
||||
let doc_id_iter: Box<dyn Iterator<Item = u32>> = if let Some(doc_id_map) = doc_id_map {
|
||||
Box::new(doc_id_map.iter_old_doc_ids())
|
||||
} else {
|
||||
let max_doc = doc_index.len() as DocId;
|
||||
Box::new(0..max_doc)
|
||||
};
|
||||
doc_id_iter.map(move |doc_id| get_values_for_doc_id(doc_id, vals, doc_index))
|
||||
}
|
||||
|
||||
/// returns all values for a doc_id
|
||||
fn get_values_for_doc_id<'a, T>(doc_id: u32, vals: &'a [T], doc_index: &'a [u64]) -> &'a [T] {
|
||||
let start_pos = doc_index[doc_id as usize] as usize;
|
||||
let end_pos = doc_index
|
||||
.get(doc_id as usize + 1)
|
||||
.cloned()
|
||||
.unwrap_or(vals.len() as u64) as usize; // special case, last doc_id has no offset information
|
||||
&vals[start_pos..end_pos]
|
||||
let offsets = doc_id_map
|
||||
.iter_old_doc_ids()
|
||||
.map(move |old_doc| {
|
||||
let num_vals_for_doc =
|
||||
column.get_val(old_doc as u64 + 1) - column.get_val(old_doc as u64);
|
||||
offset += num_vals_for_doc;
|
||||
offset
|
||||
});
|
||||
std::iter::once(0u64).chain(offsets)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
use std::net::Ipv6Addr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use fastfield_codecs::{open, open_u128, Column};
|
||||
use fastfield_codecs::{open, Column};
|
||||
|
||||
use super::multivalued::MultiValuedU128FastFieldReader;
|
||||
use crate::directory::{CompositeFile, FileSlice};
|
||||
use crate::fastfield::{
|
||||
BytesFastFieldReader, FastFieldNotAvailableError, FastValue, MultiValuedFastFieldReader,
|
||||
@@ -25,7 +23,6 @@ pub struct FastFieldReaders {
|
||||
pub(crate) enum FastType {
|
||||
I64,
|
||||
U64,
|
||||
U128,
|
||||
F64,
|
||||
Bool,
|
||||
Date,
|
||||
@@ -52,9 +49,6 @@ pub(crate) fn type_and_cardinality(field_type: &FieldType) -> Option<(FastType,
|
||||
FieldType::Str(options) if options.is_fast() => {
|
||||
Some((FastType::U64, Cardinality::MultiValues))
|
||||
}
|
||||
FieldType::IpAddr(options) => options
|
||||
.get_fastfield_cardinality()
|
||||
.map(|cardinality| (FastType::U128, cardinality)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -149,59 +143,6 @@ impl FastFieldReaders {
|
||||
self.typed_fast_field_reader(field)
|
||||
}
|
||||
|
||||
/// Returns the `ip` fast field reader reader associated to `field`.
|
||||
///
|
||||
/// If `field` is not a u128 fast field, this method returns an Error.
|
||||
pub fn ip_addr(&self, field: Field) -> crate::Result<Arc<dyn Column<Ipv6Addr>>> {
|
||||
self.check_type(field, FastType::U128, Cardinality::SingleValue)?;
|
||||
let bytes = self.fast_field_data(field, 0)?.read_bytes()?;
|
||||
Ok(open_u128::<Ipv6Addr>(bytes)?)
|
||||
}
|
||||
|
||||
/// Returns the `ip` fast field reader reader associated to `field`.
|
||||
///
|
||||
/// If `field` is not a u128 fast field, this method returns an Error.
|
||||
pub fn ip_addrs(
|
||||
&self,
|
||||
field: Field,
|
||||
) -> crate::Result<MultiValuedU128FastFieldReader<Ipv6Addr>> {
|
||||
self.check_type(field, FastType::U128, Cardinality::MultiValues)?;
|
||||
let idx_reader: Arc<dyn Column<u64>> = self.typed_fast_field_reader(field)?;
|
||||
|
||||
let bytes = self.fast_field_data(field, 1)?.read_bytes()?;
|
||||
let vals_reader = open_u128::<Ipv6Addr>(bytes)?;
|
||||
|
||||
Ok(MultiValuedU128FastFieldReader::open(
|
||||
idx_reader,
|
||||
vals_reader,
|
||||
))
|
||||
}
|
||||
|
||||
/// Returns the `u128` fast field reader reader associated to `field`.
|
||||
///
|
||||
/// If `field` is not a u128 fast field, this method returns an Error.
|
||||
pub(crate) fn u128(&self, field: Field) -> crate::Result<Arc<dyn Column<u128>>> {
|
||||
self.check_type(field, FastType::U128, Cardinality::SingleValue)?;
|
||||
let bytes = self.fast_field_data(field, 0)?.read_bytes()?;
|
||||
Ok(open_u128::<u128>(bytes)?)
|
||||
}
|
||||
|
||||
/// Returns the `u128` multi-valued fast field reader reader associated to `field`.
|
||||
///
|
||||
/// If `field` is not a u128 multi-valued fast field, this method returns an Error.
|
||||
pub fn u128s(&self, field: Field) -> crate::Result<MultiValuedU128FastFieldReader<u128>> {
|
||||
self.check_type(field, FastType::U128, Cardinality::MultiValues)?;
|
||||
let idx_reader: Arc<dyn Column<u64>> = self.typed_fast_field_reader(field)?;
|
||||
|
||||
let bytes = self.fast_field_data(field, 1)?.read_bytes()?;
|
||||
let vals_reader = open_u128::<u128>(bytes)?;
|
||||
|
||||
Ok(MultiValuedU128FastFieldReader::open(
|
||||
idx_reader,
|
||||
vals_reader,
|
||||
))
|
||||
}
|
||||
|
||||
/// Returns the `u64` fast field reader reader associated with `field`, regardless of whether
|
||||
/// the given field is effectively of type `u64` or not.
|
||||
///
|
||||
|
||||
@@ -70,35 +70,6 @@ impl CompositeFastFieldSerializer {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Serialize data into a new u64 fast field. The best compression codec of the the provided
|
||||
/// will be chosen.
|
||||
pub fn create_auto_detect_u64_fast_field_with_idx_and_codecs<T: MonotonicallyMappableToU64>(
|
||||
&mut self,
|
||||
field: Field,
|
||||
fastfield_accessor: impl Column<T>,
|
||||
idx: usize,
|
||||
codec_types: &[FastFieldCodecType],
|
||||
) -> io::Result<()> {
|
||||
let field_write = self.composite_write.for_field_with_idx(field, idx);
|
||||
fastfield_codecs::serialize(fastfield_accessor, field_write, codec_types)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Serialize data into a new u128 fast field. The codec will be compact space compressor,
|
||||
/// which is optimized for scanning the fast field for a given range.
|
||||
pub fn create_u128_fast_field_with_idx<F: Fn() -> I, I: Iterator<Item = u128>>(
|
||||
&mut self,
|
||||
field: Field,
|
||||
iter_gen: F,
|
||||
num_vals: u32,
|
||||
idx: usize,
|
||||
) -> io::Result<()> {
|
||||
let field_write = self.composite_write.for_field_with_idx(field, idx);
|
||||
fastfield_codecs::serialize_u128(iter_gen, num_vals, field_write)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Start serializing a new [u8] fast field. Use the returned writer to write data into the
|
||||
/// bytes field. To associate the bytes with documents a seperate index must be created on
|
||||
/// index 0. See bytes/writer.rs::serialize for an example.
|
||||
|
||||
@@ -2,11 +2,11 @@ use std::collections::HashMap;
|
||||
use std::io;
|
||||
|
||||
use common;
|
||||
use fastfield_codecs::{Column, MonotonicallyMappableToU128, MonotonicallyMappableToU64};
|
||||
use rustc_hash::FxHashMap;
|
||||
use fastfield_codecs::{Column, MonotonicallyMappableToU64};
|
||||
use fnv::FnvHashMap;
|
||||
use tantivy_bitpacker::BlockedBitpacker;
|
||||
|
||||
use super::multivalued::{MultiValueU128FastFieldWriter, MultiValuedFastFieldWriter};
|
||||
use super::multivalued::MultiValuedFastFieldWriter;
|
||||
use super::FastFieldType;
|
||||
use crate::fastfield::{BytesFastFieldWriter, CompositeFastFieldSerializer};
|
||||
use crate::indexer::doc_id_mapping::DocIdMapping;
|
||||
@@ -19,19 +19,10 @@ use crate::DatePrecision;
|
||||
pub struct FastFieldsWriter {
|
||||
term_id_writers: Vec<MultiValuedFastFieldWriter>,
|
||||
single_value_writers: Vec<IntFastFieldWriter>,
|
||||
u128_value_writers: Vec<U128FastFieldWriter>,
|
||||
u128_multi_value_writers: Vec<MultiValueU128FastFieldWriter>,
|
||||
multi_values_writers: Vec<MultiValuedFastFieldWriter>,
|
||||
bytes_value_writers: Vec<BytesFastFieldWriter>,
|
||||
}
|
||||
|
||||
pub(crate) fn unexpected_value(expected: &str, actual: &Value) -> crate::TantivyError {
|
||||
crate::TantivyError::SchemaError(format!(
|
||||
"Expected a {:?} in fast field, but got {:?}",
|
||||
expected, actual
|
||||
))
|
||||
}
|
||||
|
||||
fn fast_field_default_value(field_entry: &FieldEntry) -> u64 {
|
||||
match *field_entry.field_type() {
|
||||
FieldType::I64(_) | FieldType::Date(_) => common::i64_to_u64(0i64),
|
||||
@@ -43,8 +34,6 @@ fn fast_field_default_value(field_entry: &FieldEntry) -> u64 {
|
||||
impl FastFieldsWriter {
|
||||
/// Create all `FastFieldWriter` required by the schema.
|
||||
pub fn from_schema(schema: &Schema) -> FastFieldsWriter {
|
||||
let mut u128_value_writers = Vec::new();
|
||||
let mut u128_multi_value_writers = Vec::new();
|
||||
let mut single_value_writers = Vec::new();
|
||||
let mut term_id_writers = Vec::new();
|
||||
let mut multi_values_writers = Vec::new();
|
||||
@@ -108,27 +97,10 @@ impl FastFieldsWriter {
|
||||
bytes_value_writers.push(fast_field_writer);
|
||||
}
|
||||
}
|
||||
FieldType::IpAddr(opt) => {
|
||||
if opt.is_fast() {
|
||||
match opt.get_fastfield_cardinality() {
|
||||
Some(Cardinality::SingleValue) => {
|
||||
let fast_field_writer = U128FastFieldWriter::new(field);
|
||||
u128_value_writers.push(fast_field_writer);
|
||||
}
|
||||
Some(Cardinality::MultiValues) => {
|
||||
let fast_field_writer = MultiValueU128FastFieldWriter::new(field);
|
||||
u128_multi_value_writers.push(fast_field_writer);
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
FieldType::Str(_) | FieldType::JsonObject(_) => {}
|
||||
}
|
||||
}
|
||||
FastFieldsWriter {
|
||||
u128_value_writers,
|
||||
u128_multi_value_writers,
|
||||
term_id_writers,
|
||||
single_value_writers,
|
||||
multi_values_writers,
|
||||
@@ -157,16 +129,6 @@ impl FastFieldsWriter {
|
||||
.iter()
|
||||
.map(|w| w.mem_usage())
|
||||
.sum::<usize>()
|
||||
+ self
|
||||
.u128_value_writers
|
||||
.iter()
|
||||
.map(|w| w.mem_usage())
|
||||
.sum::<usize>()
|
||||
+ self
|
||||
.u128_multi_value_writers
|
||||
.iter()
|
||||
.map(|w| w.mem_usage())
|
||||
.sum::<usize>()
|
||||
}
|
||||
|
||||
/// Get the `FastFieldWriter` associated with a field.
|
||||
@@ -228,27 +190,21 @@ impl FastFieldsWriter {
|
||||
.iter_mut()
|
||||
.find(|field_writer| field_writer.field() == field)
|
||||
}
|
||||
|
||||
/// Indexes all of the fastfields of a new document.
|
||||
pub fn add_document(&mut self, doc: &Document) -> crate::Result<()> {
|
||||
pub fn add_document(&mut self, doc: &Document) {
|
||||
for field_writer in &mut self.term_id_writers {
|
||||
field_writer.add_document(doc)?;
|
||||
field_writer.add_document(doc);
|
||||
}
|
||||
for field_writer in &mut self.single_value_writers {
|
||||
field_writer.add_document(doc)?;
|
||||
field_writer.add_document(doc);
|
||||
}
|
||||
for field_writer in &mut self.multi_values_writers {
|
||||
field_writer.add_document(doc)?;
|
||||
field_writer.add_document(doc);
|
||||
}
|
||||
for field_writer in &mut self.bytes_value_writers {
|
||||
field_writer.add_document(doc)?;
|
||||
field_writer.add_document(doc);
|
||||
}
|
||||
for field_writer in &mut self.u128_value_writers {
|
||||
field_writer.add_document(doc)?;
|
||||
}
|
||||
for field_writer in &mut self.u128_multi_value_writers {
|
||||
field_writer.add_document(doc)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Serializes all of the `FastFieldWriter`s by pushing them in
|
||||
@@ -256,7 +212,7 @@ impl FastFieldsWriter {
|
||||
pub fn serialize(
|
||||
self,
|
||||
serializer: &mut CompositeFastFieldSerializer,
|
||||
mapping: &HashMap<Field, FxHashMap<UnorderedTermId, TermOrdinal>>,
|
||||
mapping: &HashMap<Field, FnvHashMap<UnorderedTermId, TermOrdinal>>,
|
||||
doc_id_map: Option<&DocIdMapping>,
|
||||
) -> io::Result<()> {
|
||||
for field_writer in self.term_id_writers {
|
||||
@@ -274,108 +230,6 @@ impl FastFieldsWriter {
|
||||
for field_writer in self.bytes_value_writers {
|
||||
field_writer.serialize(serializer, doc_id_map)?;
|
||||
}
|
||||
for field_writer in self.u128_value_writers {
|
||||
field_writer.serialize(serializer, doc_id_map)?;
|
||||
}
|
||||
for field_writer in self.u128_multi_value_writers {
|
||||
field_writer.serialize(serializer, doc_id_map)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Fast field writer for u128 values.
|
||||
/// The fast field writer just keeps the values in memory.
|
||||
///
|
||||
/// Only when the segment writer can be closed and
|
||||
/// persisted on disk, the fast field writer is
|
||||
/// sent to a `FastFieldSerializer` via the `.serialize(...)`
|
||||
/// method.
|
||||
///
|
||||
/// We cannot serialize earlier as the values are
|
||||
/// compressed to a compact number space and the number of
|
||||
/// bits required for bitpacking can only been known once
|
||||
/// we have seen all of the values.
|
||||
pub struct U128FastFieldWriter {
|
||||
field: Field,
|
||||
vals: Vec<u128>,
|
||||
val_count: u32,
|
||||
}
|
||||
|
||||
impl U128FastFieldWriter {
|
||||
/// Creates a new `IntFastFieldWriter`
|
||||
pub fn new(field: Field) -> Self {
|
||||
Self {
|
||||
field,
|
||||
vals: vec![],
|
||||
val_count: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// The memory used (inclusive childs)
|
||||
pub fn mem_usage(&self) -> usize {
|
||||
self.vals.len() * 16
|
||||
}
|
||||
|
||||
/// Records a new value.
|
||||
///
|
||||
/// The n-th value being recorded is implicitely
|
||||
/// associated to the document with the `DocId` n.
|
||||
/// (Well, `n-1` actually because of 0-indexing)
|
||||
pub fn add_val(&mut self, val: u128) {
|
||||
self.vals.push(val);
|
||||
}
|
||||
|
||||
/// Extract the fast field value from the document
|
||||
/// (or use the default value) and records it.
|
||||
///
|
||||
/// Extract the value associated to the fast field for
|
||||
/// this document.
|
||||
pub fn add_document(&mut self, doc: &Document) -> crate::Result<()> {
|
||||
match doc.get_first(self.field) {
|
||||
Some(v) => {
|
||||
let ip_addr = v.as_ip_addr().ok_or_else(|| unexpected_value("ip", v))?;
|
||||
let value = ip_addr.to_u128();
|
||||
self.add_val(value);
|
||||
}
|
||||
None => {
|
||||
self.add_val(0); // TODO fix null handling
|
||||
}
|
||||
};
|
||||
self.val_count += 1;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Push the fast fields value to the `FastFieldWriter`.
|
||||
pub fn serialize(
|
||||
&self,
|
||||
serializer: &mut CompositeFastFieldSerializer,
|
||||
doc_id_map: Option<&DocIdMapping>,
|
||||
) -> io::Result<()> {
|
||||
if let Some(doc_id_map) = doc_id_map {
|
||||
let iter_gen = || {
|
||||
doc_id_map
|
||||
.iter_old_doc_ids()
|
||||
.map(|idx| self.vals[idx as usize])
|
||||
};
|
||||
|
||||
serializer.create_u128_fast_field_with_idx(
|
||||
self.field,
|
||||
iter_gen,
|
||||
self.val_count as u32,
|
||||
0,
|
||||
)?;
|
||||
} else {
|
||||
let iter_gen = || self.vals.iter().cloned();
|
||||
serializer.create_u128_fast_field_with_idx(
|
||||
self.field,
|
||||
iter_gen,
|
||||
self.val_count as u32,
|
||||
0,
|
||||
)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -384,7 +238,7 @@ impl U128FastFieldWriter {
|
||||
/// The fast field writer just keeps the values in memory.
|
||||
///
|
||||
/// Only when the segment writer can be closed and
|
||||
/// persisted on disk, the fast field writer is
|
||||
/// persisted on disc, the fast field writer is
|
||||
/// sent to a `FastFieldSerializer` via the `.serialize(...)`
|
||||
/// method.
|
||||
///
|
||||
@@ -471,14 +325,14 @@ impl IntFastFieldWriter {
|
||||
/// only the first one is taken in account.
|
||||
///
|
||||
/// Values on text fast fields are skipped.
|
||||
pub fn add_document(&mut self, doc: &Document) -> crate::Result<()> {
|
||||
pub fn add_document(&mut self, doc: &Document) {
|
||||
match doc.get_first(self.field) {
|
||||
Some(v) => {
|
||||
let value = match (self.precision_opt, v) {
|
||||
(Some(precision), Value::Date(date_val)) => {
|
||||
date_val.truncate(precision).to_u64()
|
||||
}
|
||||
_ => super::value_to_u64(v)?,
|
||||
_ => super::value_to_u64(v),
|
||||
};
|
||||
self.add_val(value);
|
||||
}
|
||||
@@ -486,7 +340,6 @@ impl IntFastFieldWriter {
|
||||
self.add_val(self.val_if_missing);
|
||||
}
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// get iterator over the data
|
||||
@@ -511,7 +364,7 @@ impl IntFastFieldWriter {
|
||||
vals: &self.vals,
|
||||
min_value: min,
|
||||
max_value: max,
|
||||
num_vals: self.val_count as u32,
|
||||
num_vals: self.val_count as u64,
|
||||
};
|
||||
|
||||
serializer.create_auto_detect_u64_fast_field(self.field, fastfield_accessor)?;
|
||||
@@ -526,7 +379,7 @@ struct WriterFastFieldAccessProvider<'map, 'bitp> {
|
||||
vals: &'bitp BlockedBitpacker,
|
||||
min_value: u64,
|
||||
max_value: u64,
|
||||
num_vals: u32,
|
||||
num_vals: u64,
|
||||
}
|
||||
|
||||
impl<'map, 'bitp> Column for WriterFastFieldAccessProvider<'map, 'bitp> {
|
||||
@@ -538,7 +391,7 @@ impl<'map, 'bitp> Column for WriterFastFieldAccessProvider<'map, 'bitp> {
|
||||
/// # Panics
|
||||
///
|
||||
/// May panic if `doc` is greater than the index.
|
||||
fn get_val(&self, _doc: u32) -> u64 {
|
||||
fn get_val(&self, _doc: u64) -> u64 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
@@ -562,7 +415,7 @@ impl<'map, 'bitp> Column for WriterFastFieldAccessProvider<'map, 'bitp> {
|
||||
self.max_value
|
||||
}
|
||||
|
||||
fn num_vals(&self) -> u32 {
|
||||
fn num_vals(&self) -> u64 {
|
||||
self.num_vals
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,69 +0,0 @@
|
||||
pub struct FlatMapWithBuffer<T, F, Iter> {
|
||||
buffer: Vec<T>,
|
||||
fill_buffer: F,
|
||||
underlying_it: Iter,
|
||||
}
|
||||
|
||||
impl<T, F, Iter, I> Iterator for FlatMapWithBuffer<T, F, Iter>
|
||||
where
|
||||
Iter: Iterator<Item = I>,
|
||||
F: Fn(I, &mut Vec<T>),
|
||||
{
|
||||
type Item = T;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
while self.buffer.is_empty() {
|
||||
let next_el = self.underlying_it.next()?;
|
||||
(self.fill_buffer)(next_el, &mut self.buffer);
|
||||
// We will pop elements, so we reverse the buffer first.
|
||||
self.buffer.reverse();
|
||||
}
|
||||
self.buffer.pop()
|
||||
}
|
||||
}
|
||||
|
||||
pub trait FlatMapWithBufferIter: Iterator {
|
||||
/// Function similar to `flat_map`, but allows reusing a shared `Vec`.
|
||||
fn flat_map_with_buffer<F, T>(self, fill_buffer: F) -> FlatMapWithBuffer<T, F, Self>
|
||||
where
|
||||
F: Fn(Self::Item, &mut Vec<T>),
|
||||
Self: Sized,
|
||||
{
|
||||
FlatMapWithBuffer {
|
||||
buffer: Vec::with_capacity(10),
|
||||
fill_buffer,
|
||||
underlying_it: self,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: ?Sized> FlatMapWithBufferIter for T where T: Iterator {}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::indexer::flat_map_with_buffer::FlatMapWithBufferIter;
|
||||
|
||||
#[test]
|
||||
fn test_flat_map_with_buffer_empty() {
|
||||
let mut empty_iter = std::iter::empty::<usize>()
|
||||
.flat_map_with_buffer(|_val: usize, _buffer: &mut Vec<usize>| {});
|
||||
assert!(empty_iter.next().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_flat_map_with_buffer_simple() {
|
||||
let vals: Vec<usize> = (1..5)
|
||||
.flat_map_with_buffer(|val: usize, buffer: &mut Vec<usize>| buffer.extend(0..val))
|
||||
.collect();
|
||||
assert_eq!(&[0, 0, 1, 0, 1, 2, 0, 1, 2, 3], &vals[..]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_flat_map_filling_no_elements_does_not_stop_iterator() {
|
||||
let vals: Vec<usize> = [2, 0, 0, 3]
|
||||
.into_iter()
|
||||
.flat_map_with_buffer(|val: usize, buffer: &mut Vec<usize>| buffer.extend(0..val))
|
||||
.collect();
|
||||
assert_eq!(&[0, 1, 0, 1, 2], &vals[..]);
|
||||
}
|
||||
}
|
||||
@@ -803,9 +803,7 @@ impl Drop for IndexWriter {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::net::Ipv6Addr;
|
||||
|
||||
use fastfield_codecs::MonotonicallyMappableToU128;
|
||||
use proptest::prelude::*;
|
||||
use proptest::prop_oneof;
|
||||
use proptest::strategy::Strategy;
|
||||
@@ -817,13 +815,11 @@ mod tests {
|
||||
use crate::indexer::NoMergePolicy;
|
||||
use crate::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery};
|
||||
use crate::schema::{
|
||||
self, Cardinality, Facet, FacetOptions, IndexRecordOption, IpAddrOptions, NumericOptions,
|
||||
self, Cardinality, Facet, FacetOptions, IndexRecordOption, NumericOptions,
|
||||
TextFieldIndexing, TextOptions, FAST, INDEXED, STORED, STRING, TEXT,
|
||||
};
|
||||
use crate::store::DOCSTORE_CACHE_CAPACITY;
|
||||
use crate::{
|
||||
DateTime, DocAddress, Index, IndexSettings, IndexSortByField, Order, ReloadPolicy, Term,
|
||||
};
|
||||
use crate::{DocAddress, Index, IndexSettings, IndexSortByField, Order, ReloadPolicy, Term};
|
||||
|
||||
const LOREM: &str = "Doc Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do \
|
||||
eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad \
|
||||
@@ -1472,7 +1468,7 @@ mod tests {
|
||||
let fast_field_reader = segment_reader.fast_fields().u64(id_field)?;
|
||||
let in_order_alive_ids: Vec<u64> = segment_reader
|
||||
.doc_ids_alive()
|
||||
.map(|doc| fast_field_reader.get_val(doc))
|
||||
.map(|doc| fast_field_reader.get_val(doc as u64))
|
||||
.collect();
|
||||
assert_eq!(&in_order_alive_ids[..], &[9, 8, 7, 6, 5, 4, 1, 0]);
|
||||
Ok(())
|
||||
@@ -1533,7 +1529,7 @@ mod tests {
|
||||
let fast_field_reader = segment_reader.fast_fields().u64(id_field)?;
|
||||
let in_order_alive_ids: Vec<u64> = segment_reader
|
||||
.doc_ids_alive()
|
||||
.map(|doc| fast_field_reader.get_val(doc))
|
||||
.map(|doc| fast_field_reader.get_val(doc as u64))
|
||||
.collect();
|
||||
assert_eq!(&in_order_alive_ids[..], &[9, 8, 7, 6, 5, 4, 2, 0]);
|
||||
Ok(())
|
||||
@@ -1597,15 +1593,7 @@ mod tests {
|
||||
force_end_merge: bool,
|
||||
) -> crate::Result<()> {
|
||||
let mut schema_builder = schema::Schema::builder();
|
||||
let ip_field = schema_builder.add_ip_addr_field("ip", FAST | INDEXED | STORED);
|
||||
let ips_field = schema_builder.add_ip_addr_field(
|
||||
"ips",
|
||||
IpAddrOptions::default().set_fast(Cardinality::MultiValues),
|
||||
);
|
||||
let id_field = schema_builder.add_u64_field("id", FAST | INDEXED | STORED);
|
||||
let i64_field = schema_builder.add_i64_field("i64", INDEXED);
|
||||
let f64_field = schema_builder.add_f64_field("f64", INDEXED);
|
||||
let date_field = schema_builder.add_date_field("date", INDEXED);
|
||||
let bytes_field = schema_builder.add_bytes_field("bytes", FAST | INDEXED | STORED);
|
||||
let bool_field = schema_builder.add_bool_field("bool", FAST | INDEXED | STORED);
|
||||
let text_field = schema_builder.add_text_field(
|
||||
@@ -1619,7 +1607,6 @@ mod tests {
|
||||
);
|
||||
|
||||
let large_text_field = schema_builder.add_text_field("large_text_field", TEXT | STORED);
|
||||
let multi_text_fields = schema_builder.add_text_field("multi_text_fields", TEXT | STORED);
|
||||
|
||||
let multi_numbers = schema_builder.add_u64_field(
|
||||
"multi_numbers",
|
||||
@@ -1657,61 +1644,21 @@ mod tests {
|
||||
|
||||
let old_reader = index.reader()?;
|
||||
|
||||
let ip_exists = |id| id % 3 != 0; // 0 does not exist
|
||||
|
||||
let multi_text_field_text1 = "test1 test2 test3 test1 test2 test3";
|
||||
// rotate left
|
||||
let multi_text_field_text2 = "test2 test3 test1 test2 test3 test1";
|
||||
// rotate right
|
||||
let multi_text_field_text3 = "test3 test1 test2 test3 test1 test2";
|
||||
|
||||
for &op in ops {
|
||||
match op {
|
||||
IndexingOp::AddDoc { id } => {
|
||||
let facet = Facet::from(&("/cola/".to_string() + &id.to_string()));
|
||||
let ip_from_id = Ipv6Addr::from_u128(id as u128);
|
||||
|
||||
if !ip_exists(id) {
|
||||
// every 3rd doc has no ip field
|
||||
index_writer.add_document(doc!(id_field=>id,
|
||||
bytes_field => id.to_le_bytes().as_slice(),
|
||||
multi_numbers=> id,
|
||||
multi_numbers => id,
|
||||
bool_field => (id % 2u64) != 0,
|
||||
i64_field => id as i64,
|
||||
f64_field => id as f64,
|
||||
date_field => DateTime::from_timestamp_secs(id as i64),
|
||||
multi_bools => (id % 2u64) != 0,
|
||||
multi_bools => (id % 2u64) == 0,
|
||||
text_field => id.to_string(),
|
||||
facet_field => facet,
|
||||
large_text_field => LOREM,
|
||||
multi_text_fields => multi_text_field_text1,
|
||||
multi_text_fields => multi_text_field_text2,
|
||||
multi_text_fields => multi_text_field_text3,
|
||||
))?;
|
||||
} else {
|
||||
index_writer.add_document(doc!(id_field=>id,
|
||||
bytes_field => id.to_le_bytes().as_slice(),
|
||||
ip_field => ip_from_id,
|
||||
ips_field => ip_from_id,
|
||||
ips_field => ip_from_id,
|
||||
multi_numbers=> id,
|
||||
multi_numbers => id,
|
||||
bool_field => (id % 2u64) != 0,
|
||||
i64_field => id as i64,
|
||||
f64_field => id as f64,
|
||||
date_field => DateTime::from_timestamp_secs(id as i64),
|
||||
multi_bools => (id % 2u64) != 0,
|
||||
multi_bools => (id % 2u64) == 0,
|
||||
text_field => id.to_string(),
|
||||
facet_field => facet,
|
||||
large_text_field => LOREM,
|
||||
multi_text_fields => multi_text_field_text1,
|
||||
multi_text_fields => multi_text_field_text2,
|
||||
multi_text_fields => multi_text_field_text3,
|
||||
))?;
|
||||
}
|
||||
index_writer.add_document(doc!(id_field=>id,
|
||||
bytes_field => id.to_le_bytes().as_slice(),
|
||||
multi_numbers=> id,
|
||||
multi_numbers => id,
|
||||
bool_field => (id % 2u64) != 0,
|
||||
multi_bools => (id % 2u64) != 0,
|
||||
multi_bools => (id % 2u64) == 0,
|
||||
text_field => id.to_string(),
|
||||
facet_field => facet,
|
||||
large_text_field=> LOREM
|
||||
))?;
|
||||
}
|
||||
IndexingOp::DeleteDoc { id } => {
|
||||
index_writer.delete_term(Term::from_field_u64(id_field, id));
|
||||
@@ -1760,7 +1707,7 @@ mod tests {
|
||||
let ff_reader = segment_reader.fast_fields().u64(id_field).unwrap();
|
||||
segment_reader
|
||||
.doc_ids_alive()
|
||||
.map(move |doc| ff_reader.get_val(doc))
|
||||
.map(move |doc| ff_reader.get_val(doc as u64))
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -1771,7 +1718,7 @@ mod tests {
|
||||
let ff_reader = segment_reader.fast_fields().u64(id_field).unwrap();
|
||||
segment_reader
|
||||
.doc_ids_alive()
|
||||
.map(move |doc| ff_reader.get_val(doc))
|
||||
.map(move |doc| ff_reader.get_val(doc as u64))
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -1797,60 +1744,6 @@ mod tests {
|
||||
.collect::<HashSet<_>>()
|
||||
);
|
||||
|
||||
// Load all ips addr
|
||||
let ips: HashSet<Ipv6Addr> = searcher
|
||||
.segment_readers()
|
||||
.iter()
|
||||
.flat_map(|segment_reader| {
|
||||
let ff_reader = segment_reader.fast_fields().ip_addr(ip_field).unwrap();
|
||||
segment_reader.doc_ids_alive().flat_map(move |doc| {
|
||||
let val = ff_reader.get_val(doc);
|
||||
if val == Ipv6Addr::from_u128(0) {
|
||||
// TODO Fix null handling
|
||||
None
|
||||
} else {
|
||||
Some(val)
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
let expected_ips = expected_ids_and_num_occurrences
|
||||
.keys()
|
||||
.flat_map(|id| {
|
||||
if !ip_exists(*id) {
|
||||
None
|
||||
} else {
|
||||
Some(Ipv6Addr::from_u128(*id as u128))
|
||||
}
|
||||
})
|
||||
.collect::<HashSet<_>>();
|
||||
assert_eq!(ips, expected_ips);
|
||||
|
||||
let expected_ips = expected_ids_and_num_occurrences
|
||||
.keys()
|
||||
.filter_map(|id| {
|
||||
if !ip_exists(*id) {
|
||||
None
|
||||
} else {
|
||||
Some(Ipv6Addr::from_u128(*id as u128))
|
||||
}
|
||||
})
|
||||
.collect::<HashSet<_>>();
|
||||
let ips: HashSet<Ipv6Addr> = searcher
|
||||
.segment_readers()
|
||||
.iter()
|
||||
.flat_map(|segment_reader| {
|
||||
let ff_reader = segment_reader.fast_fields().ip_addrs(ips_field).unwrap();
|
||||
segment_reader.doc_ids_alive().flat_map(move |doc| {
|
||||
let mut vals = vec![];
|
||||
ff_reader.get_vals(doc, &mut vals);
|
||||
vals.into_iter().filter(|val| val.to_u128() != 0) // TODO Fix null handling
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(ips, expected_ips);
|
||||
|
||||
// multivalue fast field tests
|
||||
for segment_reader in searcher.segment_readers().iter() {
|
||||
let id_reader = segment_reader.fast_fields().u64(id_field).unwrap();
|
||||
@@ -1861,7 +1754,7 @@ mod tests {
|
||||
ff_reader.get_vals(doc, &mut vals);
|
||||
assert_eq!(vals.len(), 2);
|
||||
assert_eq!(vals[0], vals[1]);
|
||||
assert_eq!(id_reader.get_val(doc), vals[0]);
|
||||
assert_eq!(id_reader.get_val(doc as u64), vals[0]);
|
||||
|
||||
let mut bool_vals = vec![];
|
||||
bool_ff_reader.get_vals(doc, &mut bool_vals);
|
||||
@@ -1915,8 +1808,10 @@ mod tests {
|
||||
}
|
||||
}
|
||||
// test search
|
||||
let do_search = |term: &str, field| {
|
||||
let query = QueryParser::for_index(&index, vec![field])
|
||||
let my_text_field = index.schema().get_field("text_field").unwrap();
|
||||
|
||||
let do_search = |term: &str| {
|
||||
let query = QueryParser::for_index(&index, vec![my_text_field])
|
||||
.parse_query(term)
|
||||
.unwrap();
|
||||
let top_docs: Vec<(f32, DocAddress)> =
|
||||
@@ -1925,80 +1820,11 @@ mod tests {
|
||||
top_docs.iter().map(|el| el.1).collect::<Vec<_>>()
|
||||
};
|
||||
|
||||
let do_search2 = |term: Term| {
|
||||
let query = TermQuery::new(term, IndexRecordOption::Basic);
|
||||
let top_docs: Vec<(f32, DocAddress)> =
|
||||
searcher.search(&query, &TopDocs::with_limit(1000)).unwrap();
|
||||
|
||||
top_docs.iter().map(|el| el.1).collect::<Vec<_>>()
|
||||
};
|
||||
|
||||
for (existing_id, count) in &expected_ids_and_num_occurrences {
|
||||
let (existing_id, count) = (*existing_id, *count);
|
||||
let get_num_hits = |field| do_search(&existing_id.to_string(), field).len() as u64;
|
||||
assert_eq!(get_num_hits(text_field), count);
|
||||
assert_eq!(get_num_hits(i64_field), count);
|
||||
assert_eq!(get_num_hits(f64_field), count);
|
||||
assert_eq!(get_num_hits(id_field), count);
|
||||
|
||||
// Test multi text
|
||||
assert_eq!(
|
||||
do_search("\"test1 test2\"", multi_text_fields).len(),
|
||||
num_docs_expected
|
||||
);
|
||||
assert_eq!(
|
||||
do_search("\"test2 test3\"", multi_text_fields).len(),
|
||||
num_docs_expected
|
||||
);
|
||||
|
||||
// Test bytes
|
||||
let term = Term::from_field_bytes(bytes_field, existing_id.to_le_bytes().as_slice());
|
||||
assert_eq!(do_search2(term).len() as u64, count);
|
||||
|
||||
// Test date
|
||||
let term = Term::from_field_date(
|
||||
date_field,
|
||||
DateTime::from_timestamp_secs(existing_id as i64),
|
||||
);
|
||||
assert_eq!(do_search2(term).len() as u64, count);
|
||||
for (existing_id, count) in expected_ids_and_num_occurrences {
|
||||
assert_eq!(do_search(&existing_id.to_string()).len() as u64, count);
|
||||
}
|
||||
for deleted_id in deleted_ids {
|
||||
let assert_field = |field| {
|
||||
assert_eq!(do_search(&deleted_id.to_string(), field).len() as u64, 0);
|
||||
};
|
||||
assert_field(text_field);
|
||||
assert_field(f64_field);
|
||||
assert_field(i64_field);
|
||||
assert_field(id_field);
|
||||
|
||||
// Test bytes
|
||||
let term = Term::from_field_bytes(bytes_field, deleted_id.to_le_bytes().as_slice());
|
||||
assert_eq!(do_search2(term).len() as u64, 0);
|
||||
|
||||
// Test date
|
||||
let term =
|
||||
Term::from_field_date(date_field, DateTime::from_timestamp_secs(deleted_id as i64));
|
||||
assert_eq!(do_search2(term).len() as u64, 0);
|
||||
}
|
||||
// search ip address
|
||||
//
|
||||
for (existing_id, count) in &expected_ids_and_num_occurrences {
|
||||
let (existing_id, count) = (*existing_id, *count);
|
||||
if !ip_exists(existing_id) {
|
||||
continue;
|
||||
}
|
||||
let do_search_ip_field = |term: &str| do_search(term, ip_field).len() as u64;
|
||||
let ip_addr = Ipv6Addr::from_u128(existing_id as u128);
|
||||
// Test incoming ip as ipv6
|
||||
assert_eq!(do_search_ip_field(&format!("\"{}\"", ip_addr)), count);
|
||||
|
||||
let term = Term::from_field_ip_addr(ip_field, ip_addr);
|
||||
assert_eq!(do_search2(term).len() as u64, count);
|
||||
|
||||
// Test incoming ip as ipv4
|
||||
if let Some(ip_addr) = ip_addr.to_ipv4_mapped() {
|
||||
assert_eq!(do_search_ip_field(&format!("\"{}\"", ip_addr)), count);
|
||||
}
|
||||
for existing_id in deleted_ids {
|
||||
assert_eq!(do_search(&existing_id.to_string()).len(), 0);
|
||||
}
|
||||
// test facets
|
||||
for segment_reader in searcher.segment_readers().iter() {
|
||||
@@ -2012,7 +1838,7 @@ mod tests {
|
||||
facet_reader
|
||||
.facet_from_ord(facet_ords[0], &mut facet)
|
||||
.unwrap();
|
||||
let id = ff_reader.get_val(doc_id);
|
||||
let id = ff_reader.get_val(doc_id as u64);
|
||||
let facet_expected = Facet::from(&("/cola/".to_string() + &id.to_string()));
|
||||
|
||||
assert_eq!(facet, facet_expected);
|
||||
@@ -2021,36 +1847,6 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_minimal() {
|
||||
assert!(test_operation_strategy(
|
||||
&[
|
||||
IndexingOp::AddDoc { id: 23 },
|
||||
IndexingOp::AddDoc { id: 13 },
|
||||
IndexingOp::DeleteDoc { id: 13 }
|
||||
],
|
||||
true,
|
||||
false
|
||||
)
|
||||
.is_ok());
|
||||
|
||||
assert!(test_operation_strategy(
|
||||
&[
|
||||
IndexingOp::AddDoc { id: 23 },
|
||||
IndexingOp::AddDoc { id: 13 },
|
||||
IndexingOp::DeleteDoc { id: 13 }
|
||||
],
|
||||
false,
|
||||
false
|
||||
)
|
||||
.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_minimal_sort_merge() {
|
||||
assert!(test_operation_strategy(&[IndexingOp::AddDoc { id: 3 },], true, true).is_ok());
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#![proptest_config(ProptestConfig::with_cases(20))]
|
||||
#[test]
|
||||
@@ -2143,135 +1939,4 @@ mod tests {
|
||||
index_writer.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bug_1617_3() {
|
||||
assert!(test_operation_strategy(
|
||||
&[
|
||||
IndexingOp::DeleteDoc { id: 0 },
|
||||
IndexingOp::AddDoc { id: 6 },
|
||||
IndexingOp::DeleteDocQuery { id: 11 },
|
||||
IndexingOp::Commit,
|
||||
IndexingOp::Merge,
|
||||
IndexingOp::Commit,
|
||||
IndexingOp::Commit
|
||||
],
|
||||
false,
|
||||
false
|
||||
)
|
||||
.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bug_1617_2() {
|
||||
assert!(test_operation_strategy(
|
||||
&[
|
||||
IndexingOp::AddDoc { id: 13 },
|
||||
IndexingOp::DeleteDoc { id: 13 },
|
||||
IndexingOp::Commit,
|
||||
IndexingOp::AddDoc { id: 30 },
|
||||
IndexingOp::Commit,
|
||||
IndexingOp::Merge,
|
||||
],
|
||||
false,
|
||||
true
|
||||
)
|
||||
.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bug_1617() -> crate::Result<()> {
|
||||
let mut schema_builder = schema::Schema::builder();
|
||||
let id_field = schema_builder.add_u64_field("id", INDEXED);
|
||||
|
||||
let schema = schema_builder.build();
|
||||
let index = Index::builder().schema(schema).create_in_ram()?;
|
||||
let mut index_writer = index.writer_for_tests()?;
|
||||
index_writer.set_merge_policy(Box::new(NoMergePolicy));
|
||||
|
||||
let existing_id = 16u64;
|
||||
let deleted_id = 13u64;
|
||||
index_writer.add_document(doc!(
|
||||
id_field=>existing_id,
|
||||
))?;
|
||||
index_writer.add_document(doc!(
|
||||
id_field=>deleted_id,
|
||||
))?;
|
||||
index_writer.delete_term(Term::from_field_u64(id_field, deleted_id));
|
||||
index_writer.commit()?;
|
||||
|
||||
// Merge
|
||||
{
|
||||
assert!(index_writer.wait_merging_threads().is_ok());
|
||||
let mut index_writer = index.writer_for_tests()?;
|
||||
let segment_ids = index
|
||||
.searchable_segment_ids()
|
||||
.expect("Searchable segments failed.");
|
||||
index_writer.merge(&segment_ids).wait().unwrap();
|
||||
assert!(index_writer.wait_merging_threads().is_ok());
|
||||
}
|
||||
let searcher = index.reader()?.searcher();
|
||||
|
||||
let query = TermQuery::new(
|
||||
Term::from_field_u64(id_field, existing_id),
|
||||
IndexRecordOption::Basic,
|
||||
);
|
||||
let top_docs: Vec<(f32, DocAddress)> =
|
||||
searcher.search(&query, &TopDocs::with_limit(10)).unwrap();
|
||||
|
||||
assert_eq!(top_docs.len(), 1); // Fails
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bug_1618() -> crate::Result<()> {
|
||||
let mut schema_builder = schema::Schema::builder();
|
||||
let id_field = schema_builder.add_i64_field("id", INDEXED);
|
||||
|
||||
let schema = schema_builder.build();
|
||||
let index = Index::builder().schema(schema).create_in_ram()?;
|
||||
let mut index_writer = index.writer_for_tests()?;
|
||||
index_writer.set_merge_policy(Box::new(NoMergePolicy));
|
||||
|
||||
index_writer.add_document(doc!(
|
||||
id_field=>10i64,
|
||||
))?;
|
||||
index_writer.add_document(doc!(
|
||||
id_field=>30i64,
|
||||
))?;
|
||||
index_writer.commit()?;
|
||||
|
||||
// Merge
|
||||
{
|
||||
assert!(index_writer.wait_merging_threads().is_ok());
|
||||
let mut index_writer = index.writer_for_tests()?;
|
||||
let segment_ids = index
|
||||
.searchable_segment_ids()
|
||||
.expect("Searchable segments failed.");
|
||||
index_writer.merge(&segment_ids).wait().unwrap();
|
||||
assert!(index_writer.wait_merging_threads().is_ok());
|
||||
}
|
||||
let searcher = index.reader()?.searcher();
|
||||
|
||||
let query = TermQuery::new(
|
||||
Term::from_field_i64(id_field, 10i64),
|
||||
IndexRecordOption::Basic,
|
||||
);
|
||||
let top_docs: Vec<(f32, DocAddress)> =
|
||||
searcher.search(&query, &TopDocs::with_limit(10)).unwrap();
|
||||
|
||||
assert_eq!(top_docs.len(), 1); // Fails
|
||||
|
||||
let query = TermQuery::new(
|
||||
Term::from_field_i64(id_field, 30i64),
|
||||
IndexRecordOption::Basic,
|
||||
);
|
||||
let top_docs: Vec<(f32, DocAddress)> =
|
||||
searcher.search(&query, &TopDocs::with_limit(10)).unwrap();
|
||||
|
||||
assert_eq!(top_docs.len(), 1); // Fails
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use fastfield_codecs::MonotonicallyMappableToU64;
|
||||
use fnv::FnvHashMap;
|
||||
use murmurhash32::murmurhash2;
|
||||
use rustc_hash::FxHashMap;
|
||||
|
||||
use crate::fastfield::FastValue;
|
||||
use crate::postings::{IndexingContext, IndexingPosition, PostingsWriter};
|
||||
@@ -52,7 +52,7 @@ use crate::{DatePrecision, DateTime, DocId, Term};
|
||||
/// path map to the same index position as long as the probability is relatively low.
|
||||
#[derive(Default)]
|
||||
struct IndexingPositionsPerPath {
|
||||
positions_per_path: FxHashMap<u32, IndexingPosition>,
|
||||
positions_per_path: FnvHashMap<u32, IndexingPosition>,
|
||||
}
|
||||
|
||||
impl IndexingPositionsPerPath {
|
||||
@@ -242,12 +242,10 @@ pub(crate) fn set_string_and_get_terms(
|
||||
) -> Vec<(usize, Term)> {
|
||||
let mut positions_and_terms = Vec::<(usize, Term)>::new();
|
||||
json_term_writer.close_path_and_set_type(Type::Str);
|
||||
let term_num_bytes = json_term_writer.term_buffer.len_bytes();
|
||||
let term_num_bytes = json_term_writer.term_buffer.as_slice().len();
|
||||
let mut token_stream = text_analyzer.token_stream(value);
|
||||
token_stream.process(&mut |token| {
|
||||
json_term_writer
|
||||
.term_buffer
|
||||
.truncate_value_bytes(term_num_bytes);
|
||||
json_term_writer.term_buffer.truncate(term_num_bytes);
|
||||
json_term_writer
|
||||
.term_buffer
|
||||
.append_bytes(token.text.as_bytes());
|
||||
@@ -267,7 +265,7 @@ impl<'a> JsonTermWriter<'a> {
|
||||
json_path: &str,
|
||||
term_buffer: &'a mut Term,
|
||||
) -> Self {
|
||||
term_buffer.set_field_and_type(field, Type::Json);
|
||||
term_buffer.set_field(Type::Json, field);
|
||||
let mut json_term_writer = Self::wrap(term_buffer);
|
||||
for segment in json_path.split('.') {
|
||||
json_term_writer.push_path_segment(segment);
|
||||
@@ -278,7 +276,7 @@ impl<'a> JsonTermWriter<'a> {
|
||||
pub fn wrap(term_buffer: &'a mut Term) -> Self {
|
||||
term_buffer.clear_with_type(Type::Json);
|
||||
let mut path_stack = Vec::with_capacity(10);
|
||||
path_stack.push(0);
|
||||
path_stack.push(5);
|
||||
Self {
|
||||
term_buffer,
|
||||
path_stack,
|
||||
@@ -287,28 +285,28 @@ impl<'a> JsonTermWriter<'a> {
|
||||
|
||||
fn trim_to_end_of_path(&mut self) {
|
||||
let end_of_path = *self.path_stack.last().unwrap();
|
||||
self.term_buffer.truncate_value_bytes(end_of_path);
|
||||
self.term_buffer.truncate(end_of_path);
|
||||
}
|
||||
|
||||
pub fn close_path_and_set_type(&mut self, typ: Type) {
|
||||
self.trim_to_end_of_path();
|
||||
let buffer = self.term_buffer.value_bytes_mut();
|
||||
let buffer = self.term_buffer.as_mut();
|
||||
let buffer_len = buffer.len();
|
||||
buffer[buffer_len - 1] = JSON_END_OF_PATH;
|
||||
self.term_buffer.append_bytes(&[typ.to_code()]);
|
||||
buffer.push(typ.to_code());
|
||||
}
|
||||
|
||||
pub fn push_path_segment(&mut self, segment: &str) {
|
||||
// the path stack should never be empty.
|
||||
self.trim_to_end_of_path();
|
||||
let buffer = self.term_buffer.value_bytes_mut();
|
||||
let buffer = self.term_buffer.as_mut();
|
||||
let buffer_len = buffer.len();
|
||||
if self.path_stack.len() > 1 {
|
||||
buffer[buffer_len - 1] = JSON_PATH_SEGMENT_SEP;
|
||||
}
|
||||
self.term_buffer.append_bytes(segment.as_bytes());
|
||||
self.term_buffer.append_bytes(&[JSON_PATH_SEGMENT_SEP]);
|
||||
self.path_stack.push(self.term_buffer.len_bytes());
|
||||
buffer.extend(segment.as_bytes());
|
||||
buffer.push(JSON_PATH_SEGMENT_SEP);
|
||||
self.path_stack.push(buffer.len());
|
||||
}
|
||||
|
||||
pub fn pop_path_segment(&mut self) {
|
||||
@@ -320,8 +318,8 @@ impl<'a> JsonTermWriter<'a> {
|
||||
/// Returns the json path of the term being currently built.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn path(&self) -> &[u8] {
|
||||
let end_of_path = self.path_stack.last().cloned().unwrap_or(1);
|
||||
&self.term().value_bytes()[..end_of_path - 1]
|
||||
let end_of_path = self.path_stack.last().cloned().unwrap_or(6);
|
||||
&self.term().as_slice()[5..end_of_path - 1]
|
||||
}
|
||||
|
||||
pub fn set_fast_value<T: FastValue>(&mut self, val: T) {
|
||||
@@ -334,13 +332,14 @@ impl<'a> JsonTermWriter<'a> {
|
||||
val.to_u64()
|
||||
};
|
||||
self.term_buffer
|
||||
.append_bytes(value.to_be_bytes().as_slice());
|
||||
.as_mut()
|
||||
.extend_from_slice(value.to_be_bytes().as_slice());
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_str(&mut self, text: &str) {
|
||||
self.close_path_and_set_type(Type::Str);
|
||||
self.term_buffer.append_bytes(text.as_bytes());
|
||||
self.term_buffer.as_mut().extend_from_slice(text.as_bytes());
|
||||
}
|
||||
|
||||
pub fn term(&self) -> &Term {
|
||||
@@ -357,7 +356,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_json_writer() {
|
||||
let field = Field::from_field_id(1);
|
||||
let mut term = Term::with_type_and_field(Type::Json, field);
|
||||
let mut term = Term::new();
|
||||
term.set_field(Type::Json, field);
|
||||
let mut json_writer = JsonTermWriter::wrap(&mut term);
|
||||
json_writer.push_path_segment("attributes");
|
||||
json_writer.push_path_segment("color");
|
||||
@@ -391,7 +391,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_string_term() {
|
||||
let field = Field::from_field_id(1);
|
||||
let mut term = Term::with_type_and_field(Type::Json, field);
|
||||
let mut term = Term::new();
|
||||
term.set_field(Type::Json, field);
|
||||
let mut json_writer = JsonTermWriter::wrap(&mut term);
|
||||
json_writer.push_path_segment("color");
|
||||
json_writer.set_str("red");
|
||||
@@ -404,7 +405,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_i64_term() {
|
||||
let field = Field::from_field_id(1);
|
||||
let mut term = Term::with_type_and_field(Type::Json, field);
|
||||
let mut term = Term::new();
|
||||
term.set_field(Type::Json, field);
|
||||
let mut json_writer = JsonTermWriter::wrap(&mut term);
|
||||
json_writer.push_path_segment("color");
|
||||
json_writer.set_fast_value(-4i64);
|
||||
@@ -417,7 +419,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_u64_term() {
|
||||
let field = Field::from_field_id(1);
|
||||
let mut term = Term::with_type_and_field(Type::Json, field);
|
||||
let mut term = Term::new();
|
||||
term.set_field(Type::Json, field);
|
||||
let mut json_writer = JsonTermWriter::wrap(&mut term);
|
||||
json_writer.push_path_segment("color");
|
||||
json_writer.set_fast_value(4u64);
|
||||
@@ -430,7 +433,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_f64_term() {
|
||||
let field = Field::from_field_id(1);
|
||||
let mut term = Term::with_type_and_field(Type::Json, field);
|
||||
let mut term = Term::new();
|
||||
term.set_field(Type::Json, field);
|
||||
let mut json_writer = JsonTermWriter::wrap(&mut term);
|
||||
json_writer.push_path_segment("color");
|
||||
json_writer.set_fast_value(4.0f64);
|
||||
@@ -443,7 +447,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_bool_term() {
|
||||
let field = Field::from_field_id(1);
|
||||
let mut term = Term::with_type_and_field(Type::Json, field);
|
||||
let mut term = Term::new();
|
||||
term.set_field(Type::Json, field);
|
||||
let mut json_writer = JsonTermWriter::wrap(&mut term);
|
||||
json_writer.push_path_segment("color");
|
||||
json_writer.set_fast_value(true);
|
||||
@@ -456,7 +461,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_push_after_set_path_segment() {
|
||||
let field = Field::from_field_id(1);
|
||||
let mut term = Term::with_type_and_field(Type::Json, field);
|
||||
let mut term = Term::new();
|
||||
term.set_field(Type::Json, field);
|
||||
let mut json_writer = JsonTermWriter::wrap(&mut term);
|
||||
json_writer.push_path_segment("attribute");
|
||||
json_writer.set_str("something");
|
||||
@@ -471,7 +477,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_pop_segment() {
|
||||
let field = Field::from_field_id(1);
|
||||
let mut term = Term::with_type_and_field(Type::Json, field);
|
||||
let mut term = Term::new();
|
||||
term.set_field(Type::Json, field);
|
||||
let mut json_writer = JsonTermWriter::wrap(&mut term);
|
||||
json_writer.push_path_segment("color");
|
||||
json_writer.push_path_segment("hue");
|
||||
@@ -486,7 +493,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_json_writer_path() {
|
||||
let field = Field::from_field_id(1);
|
||||
let mut term = Term::with_type_and_field(Type::Json, field);
|
||||
let mut term = Term::new();
|
||||
term.set_field(Type::Json, field);
|
||||
let mut json_writer = JsonTermWriter::wrap(&mut term);
|
||||
json_writer.push_path_segment("color");
|
||||
assert_eq!(json_writer.path(), b"color");
|
||||
|
||||
@@ -6,14 +6,11 @@ use fastfield_codecs::VecColumn;
|
||||
use itertools::Itertools;
|
||||
use measure_time::debug_time;
|
||||
|
||||
use super::flat_map_with_buffer::FlatMapWithBufferIter;
|
||||
use super::sorted_doc_id_multivalue_column::RemappedDocIdMultiValueIndexColumn;
|
||||
use crate::core::{Segment, SegmentReader};
|
||||
use crate::docset::{DocSet, TERMINATED};
|
||||
use crate::error::DataCorruption;
|
||||
use crate::fastfield::{
|
||||
get_fastfield_codecs_for_multivalue, AliveBitSet, Column, CompositeFastFieldSerializer,
|
||||
MultiValueLength, MultiValuedFastFieldReader, MultiValuedU128FastFieldReader,
|
||||
AliveBitSet, Column, CompositeFastFieldSerializer, MultiValueLength, MultiValuedFastFieldReader,
|
||||
};
|
||||
use crate::fieldnorm::{FieldNormReader, FieldNormReaders, FieldNormsSerializer, FieldNormsWriter};
|
||||
use crate::indexer::doc_id_mapping::{expect_field_id_for_sort_field, SegmentDocIdMapping};
|
||||
@@ -296,24 +293,6 @@ impl IndexMerger {
|
||||
self.write_bytes_fast_field(field, fast_field_serializer, doc_id_mapping)?;
|
||||
}
|
||||
}
|
||||
FieldType::IpAddr(options) => match options.get_fastfield_cardinality() {
|
||||
Some(Cardinality::SingleValue) => {
|
||||
self.write_u128_single_fast_field(
|
||||
field,
|
||||
fast_field_serializer,
|
||||
doc_id_mapping,
|
||||
)?;
|
||||
}
|
||||
Some(Cardinality::MultiValues) => {
|
||||
self.write_u128_multi_fast_field(
|
||||
field,
|
||||
fast_field_serializer,
|
||||
doc_id_mapping,
|
||||
)?;
|
||||
}
|
||||
None => {}
|
||||
},
|
||||
|
||||
FieldType::JsonObject(_) | FieldType::Facet(_) | FieldType::Str(_) => {
|
||||
// We don't handle json fast field for the moment
|
||||
// They can be implemented using what is done
|
||||
@@ -324,91 +303,6 @@ impl IndexMerger {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// used to merge `u128` single fast fields.
|
||||
fn write_u128_multi_fast_field(
|
||||
&self,
|
||||
field: Field,
|
||||
fast_field_serializer: &mut CompositeFastFieldSerializer,
|
||||
doc_id_mapping: &SegmentDocIdMapping,
|
||||
) -> crate::Result<()> {
|
||||
let segment_and_ff_readers: Vec<(&SegmentReader, MultiValuedU128FastFieldReader<u128>)> =
|
||||
self.readers
|
||||
.iter()
|
||||
.map(|segment_reader| {
|
||||
let ff_reader: MultiValuedU128FastFieldReader<u128> =
|
||||
segment_reader.fast_fields().u128s(field).expect(
|
||||
"Failed to find index for multivalued field. This is a bug in \
|
||||
tantivy, please report.",
|
||||
);
|
||||
(segment_reader, ff_reader)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
Self::write_1_n_fast_field_idx_generic(
|
||||
field,
|
||||
fast_field_serializer,
|
||||
doc_id_mapping,
|
||||
&segment_and_ff_readers,
|
||||
)?;
|
||||
|
||||
let fast_field_readers = segment_and_ff_readers
|
||||
.into_iter()
|
||||
.map(|(_, ff_reader)| ff_reader)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let iter_gen = || {
|
||||
doc_id_mapping
|
||||
.iter_old_doc_addrs()
|
||||
.flat_map_with_buffer(|doc_addr, buffer| {
|
||||
let fast_field_reader = &fast_field_readers[doc_addr.segment_ord as usize];
|
||||
fast_field_reader.get_vals(doc_addr.doc_id, buffer);
|
||||
})
|
||||
};
|
||||
|
||||
fast_field_serializer.create_u128_fast_field_with_idx(
|
||||
field,
|
||||
iter_gen,
|
||||
doc_id_mapping.len() as u32,
|
||||
1,
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// used to merge `u128` single fast fields.
|
||||
fn write_u128_single_fast_field(
|
||||
&self,
|
||||
field: Field,
|
||||
fast_field_serializer: &mut CompositeFastFieldSerializer,
|
||||
doc_id_mapping: &SegmentDocIdMapping,
|
||||
) -> crate::Result<()> {
|
||||
let fast_field_readers = self
|
||||
.readers
|
||||
.iter()
|
||||
.map(|reader| {
|
||||
let u128_reader: Arc<dyn Column<u128>> = reader.fast_fields().u128(field).expect(
|
||||
"Failed to find a reader for single fast field. This is a tantivy bug and it \
|
||||
should never happen.",
|
||||
);
|
||||
u128_reader
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let iter_gen = || {
|
||||
doc_id_mapping.iter_old_doc_addrs().map(|doc_addr| {
|
||||
let fast_field_reader = &fast_field_readers[doc_addr.segment_ord as usize];
|
||||
fast_field_reader.get_val(doc_addr.doc_id)
|
||||
})
|
||||
};
|
||||
fast_field_serializer.create_u128_fast_field_with_idx(
|
||||
field,
|
||||
iter_gen,
|
||||
doc_id_mapping.len() as u32,
|
||||
0,
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// used both to merge field norms, `u64/i64` single fast fields.
|
||||
fn write_single_fast_field(
|
||||
&self,
|
||||
@@ -510,8 +404,8 @@ impl IndexMerger {
|
||||
doc_id_reader_pair
|
||||
.into_iter()
|
||||
.kmerge_by(|a, b| {
|
||||
let val1 = a.2.get_val(a.0);
|
||||
let val2 = b.2.get_val(b.0);
|
||||
let val1 = a.2.get_val(a.0 as u64);
|
||||
let val2 = b.2.get_val(b.0 as u64);
|
||||
if sort_by_field.order == Order::Asc {
|
||||
val1 < val2
|
||||
} else {
|
||||
@@ -529,16 +423,26 @@ impl IndexMerger {
|
||||
// Creating the index file to point into the data, generic over `BytesFastFieldReader` and
|
||||
// `MultiValuedFastFieldReader`
|
||||
//
|
||||
fn write_1_n_fast_field_idx_generic<T: MultiValueLength + Send + Sync>(
|
||||
fn write_1_n_fast_field_idx_generic<T: MultiValueLength>(
|
||||
field: Field,
|
||||
fast_field_serializer: &mut CompositeFastFieldSerializer,
|
||||
doc_id_mapping: &SegmentDocIdMapping,
|
||||
segment_and_ff_readers: &[(&SegmentReader, T)],
|
||||
reader_and_field_accessors: &[(&SegmentReader, T)],
|
||||
) -> crate::Result<()> {
|
||||
let column =
|
||||
RemappedDocIdMultiValueIndexColumn::new(segment_and_ff_readers, doc_id_mapping);
|
||||
// TODO Use `Column` implementation instead
|
||||
|
||||
fast_field_serializer.create_auto_detect_u64_fast_field(field, column)?;
|
||||
let mut offsets = Vec::with_capacity(doc_id_mapping.len());
|
||||
let mut offset = 0;
|
||||
for old_doc_addr in doc_id_mapping.iter_old_doc_addrs() {
|
||||
let reader = &reader_and_field_accessors[old_doc_addr.segment_ord as usize].1;
|
||||
offsets.push(offset);
|
||||
offset += reader.get_len(old_doc_addr.doc_id) as u64;
|
||||
}
|
||||
offsets.push(offset);
|
||||
|
||||
let fastfield_accessor = VecColumn::from(&offsets[..]);
|
||||
|
||||
fast_field_serializer.create_auto_detect_u64_fast_field(field, fastfield_accessor)?;
|
||||
Ok(())
|
||||
}
|
||||
/// Returns the fastfield index (index for the data, not the data).
|
||||
@@ -548,7 +452,7 @@ impl IndexMerger {
|
||||
fast_field_serializer: &mut CompositeFastFieldSerializer,
|
||||
doc_id_mapping: &SegmentDocIdMapping,
|
||||
) -> crate::Result<()> {
|
||||
let segment_and_ff_readers = self
|
||||
let reader_ordinal_and_field_accessors = self
|
||||
.readers
|
||||
.iter()
|
||||
.map(|reader| {
|
||||
@@ -567,7 +471,7 @@ impl IndexMerger {
|
||||
field,
|
||||
fast_field_serializer,
|
||||
doc_id_mapping,
|
||||
&segment_and_ff_readers,
|
||||
&reader_ordinal_and_field_accessors,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -616,12 +520,7 @@ impl IndexMerger {
|
||||
}
|
||||
|
||||
let col = VecColumn::from(&vals[..]);
|
||||
fast_field_serializer.create_auto_detect_u64_fast_field_with_idx_and_codecs(
|
||||
field,
|
||||
col,
|
||||
1,
|
||||
&get_fastfield_codecs_for_multivalue(),
|
||||
)?;
|
||||
fast_field_serializer.create_auto_detect_u64_fast_field_with_idx(field, col, 1)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -666,11 +565,10 @@ impl IndexMerger {
|
||||
|
||||
let fastfield_accessor =
|
||||
RemappedDocIdMultiValueColumn::new(&self.readers, doc_id_mapping, field);
|
||||
fast_field_serializer.create_auto_detect_u64_fast_field_with_idx_and_codecs(
|
||||
fast_field_serializer.create_auto_detect_u64_fast_field_with_idx(
|
||||
field,
|
||||
fastfield_accessor,
|
||||
1,
|
||||
&get_fastfield_codecs_for_multivalue(),
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
@@ -682,7 +580,7 @@ impl IndexMerger {
|
||||
fast_field_serializer: &mut CompositeFastFieldSerializer,
|
||||
doc_id_mapping: &SegmentDocIdMapping,
|
||||
) -> crate::Result<()> {
|
||||
let segment_and_ff_readers = self
|
||||
let reader_and_field_accessors = self
|
||||
.readers
|
||||
.iter()
|
||||
.map(|reader| {
|
||||
@@ -693,17 +591,17 @@ impl IndexMerger {
|
||||
(reader, bytes_reader)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
Self::write_1_n_fast_field_idx_generic(
|
||||
field,
|
||||
fast_field_serializer,
|
||||
doc_id_mapping,
|
||||
&segment_and_ff_readers,
|
||||
&reader_and_field_accessors,
|
||||
)?;
|
||||
|
||||
let mut serialize_vals = fast_field_serializer.new_bytes_fast_field(field);
|
||||
|
||||
for old_doc_addr in doc_id_mapping.iter_old_doc_addrs() {
|
||||
let bytes_reader = &segment_and_ff_readers[old_doc_addr.segment_ord as usize].1;
|
||||
let bytes_reader = &reader_and_field_accessors[old_doc_addr.segment_ord as usize].1;
|
||||
let val = bytes_reader.get_bytes(old_doc_addr.doc_id);
|
||||
serialize_vals.write_all(val)?;
|
||||
}
|
||||
|
||||
@@ -190,13 +190,13 @@ mod tests {
|
||||
assert_eq!(fast_field.get_val(4), 2u64);
|
||||
assert_eq!(fast_field.get_val(3), 3u64);
|
||||
if force_disjunct_segment_sort_values {
|
||||
assert_eq!(fast_field.get_val(2), 20u64);
|
||||
assert_eq!(fast_field.get_val(1), 100u64);
|
||||
assert_eq!(fast_field.get_val(2u64), 20u64);
|
||||
assert_eq!(fast_field.get_val(1u64), 100u64);
|
||||
} else {
|
||||
assert_eq!(fast_field.get_val(2), 10u64);
|
||||
assert_eq!(fast_field.get_val(1), 20u64);
|
||||
assert_eq!(fast_field.get_val(2u64), 10u64);
|
||||
assert_eq!(fast_field.get_val(1u64), 20u64);
|
||||
}
|
||||
assert_eq!(fast_field.get_val(0), 1_000u64);
|
||||
assert_eq!(fast_field.get_val(0u64), 1_000u64);
|
||||
|
||||
// test new field norm mapping
|
||||
{
|
||||
@@ -545,7 +545,7 @@ mod bench_sorted_index_merge {
|
||||
// add values in order of the new doc_ids
|
||||
let mut val = 0;
|
||||
for (doc_id, _reader, field_reader) in sorted_doc_ids {
|
||||
val = field_reader.get_val(doc_id);
|
||||
val = field_reader.get_val(doc_id as u64);
|
||||
}
|
||||
|
||||
val
|
||||
|
||||
@@ -3,7 +3,6 @@ pub mod delete_queue;
|
||||
pub mod demuxer;
|
||||
pub mod doc_id_mapping;
|
||||
mod doc_opstamp_mapping;
|
||||
mod flat_map_with_buffer;
|
||||
pub mod index_writer;
|
||||
mod index_writer_status;
|
||||
mod json_term_writer;
|
||||
|
||||
@@ -30,10 +30,8 @@ impl SegmentSerializer {
|
||||
StoreWriter::new(
|
||||
store_write,
|
||||
crate::store::Compressor::None,
|
||||
// We want fast random access on the docs, so we choose a small block size.
|
||||
// If this is zero, the skip index will contain too many checkpoints and
|
||||
// therefore will be relatively slow.
|
||||
16000,
|
||||
0, // we want random access on the docs, so we choose a minimal block size. Every
|
||||
// doc will get its own block.
|
||||
settings.docstore_compress_dedicated_thread,
|
||||
)?
|
||||
} else {
|
||||
|
||||
@@ -133,15 +133,15 @@ fn merge(
|
||||
|
||||
/// Advanced: Merges a list of segments from different indices in a new index.
|
||||
///
|
||||
/// Returns `TantivyError` if the indices list is empty or their
|
||||
/// Returns `TantivyError` if the the indices list is empty or their
|
||||
/// schemas don't match.
|
||||
///
|
||||
/// `output_directory`: is assumed to be empty.
|
||||
///
|
||||
/// # Warning
|
||||
/// This function does NOT check or take the `IndexWriter` is running. It is not
|
||||
/// meant to work if you have an `IndexWriter` running for the origin indices, or
|
||||
/// the destination `Index`.
|
||||
/// meant to work if you have an IndexWriter running for the origin indices, or
|
||||
/// the destination Index.
|
||||
#[doc(hidden)]
|
||||
pub fn merge_indices<T: Into<Box<dyn Directory>>>(
|
||||
indices: &[Index],
|
||||
@@ -179,15 +179,15 @@ pub fn merge_indices<T: Into<Box<dyn Directory>>>(
|
||||
/// Advanced: Merges a list of segments from different indices in a new index.
|
||||
/// Additional you can provide a delete bitset for each segment to ignore doc_ids.
|
||||
///
|
||||
/// Returns `TantivyError` if the indices list is empty or their
|
||||
/// Returns `TantivyError` if the the indices list is empty or their
|
||||
/// schemas don't match.
|
||||
///
|
||||
/// `output_directory`: is assumed to be empty.
|
||||
///
|
||||
/// # Warning
|
||||
/// This function does NOT check or take the `IndexWriter` is running. It is not
|
||||
/// meant to work if you have an `IndexWriter` running for the origin indices, or
|
||||
/// the destination `Index`.
|
||||
/// meant to work if you have an IndexWriter running for the origin indices, or
|
||||
/// the destination Index.
|
||||
#[doc(hidden)]
|
||||
pub fn merge_filtered_segments<T: Into<Box<dyn Directory>>>(
|
||||
segments: &[Segment],
|
||||
|
||||
@@ -12,9 +12,11 @@ use crate::postings::{
|
||||
compute_table_size, serialize_postings, IndexingContext, IndexingPosition,
|
||||
PerFieldPostingsWriter, PostingsWriter,
|
||||
};
|
||||
use crate::schema::{FieldEntry, FieldType, Schema, Term, Value};
|
||||
use crate::schema::{FieldEntry, FieldType, FieldValue, Schema, Term, Value};
|
||||
use crate::store::{StoreReader, StoreWriter};
|
||||
use crate::tokenizer::{FacetTokenizer, PreTokenizedStream, TextAnalyzer, Tokenizer};
|
||||
use crate::tokenizer::{
|
||||
BoxTokenStream, FacetTokenizer, PreTokenizedStream, TextAnalyzer, Tokenizer,
|
||||
};
|
||||
use crate::{DatePrecision, DocId, Document, Opstamp, SegmentComponent};
|
||||
|
||||
/// Computes the initial size of the hash table.
|
||||
@@ -114,7 +116,7 @@ impl SegmentWriter {
|
||||
fast_field_writers: FastFieldsWriter::from_schema(&schema),
|
||||
doc_opstamps: Vec::with_capacity(1_000),
|
||||
per_field_text_analyzers,
|
||||
term_buffer: Term::with_capacity(16),
|
||||
term_buffer: Term::new(),
|
||||
schema,
|
||||
})
|
||||
}
|
||||
@@ -158,6 +160,7 @@ impl SegmentWriter {
|
||||
let doc_id = self.max_doc;
|
||||
let vals_grouped_by_field = doc
|
||||
.field_values()
|
||||
.iter()
|
||||
.sorted_by_key(|el| el.field())
|
||||
.group_by(|el| el.field());
|
||||
for (field, field_values) in &vals_grouped_by_field {
|
||||
@@ -173,12 +176,10 @@ impl SegmentWriter {
|
||||
if !field_entry.is_indexed() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let (term_buffer, ctx) = (&mut self.term_buffer, &mut self.ctx);
|
||||
let postings_writer: &mut dyn PostingsWriter =
|
||||
self.per_field_postings_writers.get_for_field_mut(field);
|
||||
term_buffer.clear_with_field_and_type(field_entry.field_type().value_type(), field);
|
||||
|
||||
term_buffer.set_field(field_entry.field_type().value_type(), field);
|
||||
match *field_entry.field_type() {
|
||||
FieldType::Facet(_) => {
|
||||
for value in values {
|
||||
@@ -203,23 +204,27 @@ impl SegmentWriter {
|
||||
}
|
||||
}
|
||||
FieldType::Str(_) => {
|
||||
let mut indexing_position = IndexingPosition::default();
|
||||
let mut token_streams: Vec<BoxTokenStream> = vec![];
|
||||
|
||||
for value in values {
|
||||
let mut token_stream = match value {
|
||||
match value {
|
||||
Value::PreTokStr(tok_str) => {
|
||||
PreTokenizedStream::from(tok_str.clone()).into()
|
||||
token_streams
|
||||
.push(PreTokenizedStream::from(tok_str.clone()).into());
|
||||
}
|
||||
Value::Str(ref text) => {
|
||||
let text_analyzer =
|
||||
&self.per_field_text_analyzers[field.field_id() as usize];
|
||||
text_analyzer.token_stream(text)
|
||||
token_streams.push(text_analyzer.token_stream(text));
|
||||
}
|
||||
_ => {
|
||||
continue;
|
||||
}
|
||||
};
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
assert!(term_buffer.is_empty());
|
||||
let mut indexing_position = IndexingPosition::default();
|
||||
|
||||
for mut token_stream in token_streams {
|
||||
assert_eq!(term_buffer.as_slice().len(), 5);
|
||||
postings_writer.index_text(
|
||||
doc_id,
|
||||
&mut *token_stream,
|
||||
@@ -235,76 +240,46 @@ impl SegmentWriter {
|
||||
}
|
||||
}
|
||||
FieldType::U64(_) => {
|
||||
let mut num_vals = 0;
|
||||
for value in values {
|
||||
num_vals += 1;
|
||||
let u64_val = value.as_u64().ok_or_else(make_schema_error)?;
|
||||
term_buffer.set_u64(u64_val);
|
||||
postings_writer.subscribe(doc_id, 0u32, term_buffer, ctx);
|
||||
}
|
||||
if field_entry.has_fieldnorms() {
|
||||
self.fieldnorms_writer.record(doc_id, field, num_vals);
|
||||
}
|
||||
}
|
||||
FieldType::Date(_) => {
|
||||
let mut num_vals = 0;
|
||||
for value in values {
|
||||
num_vals += 1;
|
||||
let date_val = value.as_date().ok_or_else(make_schema_error)?;
|
||||
term_buffer.set_u64(date_val.truncate(DatePrecision::Seconds).to_u64());
|
||||
postings_writer.subscribe(doc_id, 0u32, term_buffer, ctx);
|
||||
}
|
||||
if field_entry.has_fieldnorms() {
|
||||
self.fieldnorms_writer.record(doc_id, field, num_vals);
|
||||
}
|
||||
}
|
||||
FieldType::I64(_) => {
|
||||
let mut num_vals = 0;
|
||||
for value in values {
|
||||
num_vals += 1;
|
||||
let i64_val = value.as_i64().ok_or_else(make_schema_error)?;
|
||||
term_buffer.set_i64(i64_val);
|
||||
postings_writer.subscribe(doc_id, 0u32, term_buffer, ctx);
|
||||
}
|
||||
if field_entry.has_fieldnorms() {
|
||||
self.fieldnorms_writer.record(doc_id, field, num_vals);
|
||||
}
|
||||
}
|
||||
FieldType::F64(_) => {
|
||||
let mut num_vals = 0;
|
||||
for value in values {
|
||||
num_vals += 1;
|
||||
let f64_val = value.as_f64().ok_or_else(make_schema_error)?;
|
||||
term_buffer.set_f64(f64_val);
|
||||
postings_writer.subscribe(doc_id, 0u32, term_buffer, ctx);
|
||||
}
|
||||
if field_entry.has_fieldnorms() {
|
||||
self.fieldnorms_writer.record(doc_id, field, num_vals);
|
||||
}
|
||||
}
|
||||
FieldType::Bool(_) => {
|
||||
let mut num_vals = 0;
|
||||
for value in values {
|
||||
num_vals += 1;
|
||||
let bool_val = value.as_bool().ok_or_else(make_schema_error)?;
|
||||
term_buffer.set_bool(bool_val);
|
||||
postings_writer.subscribe(doc_id, 0u32, term_buffer, ctx);
|
||||
}
|
||||
if field_entry.has_fieldnorms() {
|
||||
self.fieldnorms_writer.record(doc_id, field, num_vals);
|
||||
}
|
||||
}
|
||||
FieldType::Bytes(_) => {
|
||||
let mut num_vals = 0;
|
||||
for value in values {
|
||||
num_vals += 1;
|
||||
let bytes = value.as_bytes().ok_or_else(make_schema_error)?;
|
||||
term_buffer.set_bytes(bytes);
|
||||
postings_writer.subscribe(doc_id, 0u32, term_buffer, ctx);
|
||||
}
|
||||
if field_entry.has_fieldnorms() {
|
||||
self.fieldnorms_writer.record(doc_id, field, num_vals);
|
||||
}
|
||||
}
|
||||
FieldType::JsonObject(_) => {
|
||||
let text_analyzer = &self.per_field_text_analyzers[field.field_id() as usize];
|
||||
@@ -319,18 +294,6 @@ impl SegmentWriter {
|
||||
ctx,
|
||||
)?;
|
||||
}
|
||||
FieldType::IpAddr(_) => {
|
||||
let mut num_vals = 0;
|
||||
for value in values {
|
||||
num_vals += 1;
|
||||
let ip_addr = value.as_ip_addr().ok_or_else(make_schema_error)?;
|
||||
term_buffer.set_ip_addr(ip_addr);
|
||||
postings_writer.subscribe(doc_id, 0u32, term_buffer, ctx);
|
||||
}
|
||||
if field_entry.has_fieldnorms() {
|
||||
self.fieldnorms_writer.record(doc_id, field, num_vals);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -342,10 +305,11 @@ impl SegmentWriter {
|
||||
pub fn add_document(&mut self, add_operation: AddOperation) -> crate::Result<()> {
|
||||
let doc = add_operation.document;
|
||||
self.doc_opstamps.push(add_operation.opstamp);
|
||||
self.fast_field_writers.add_document(&doc)?;
|
||||
self.fast_field_writers.add_document(&doc);
|
||||
self.index_document(&doc)?;
|
||||
let prepared_doc = prepare_doc_for_store(doc, &self.schema);
|
||||
let doc_writer = self.segment_serializer.get_store_writer();
|
||||
doc_writer.store(&doc, &self.schema)?;
|
||||
doc_writer.store(&prepared_doc)?;
|
||||
self.max_doc += 1;
|
||||
Ok(())
|
||||
}
|
||||
@@ -442,24 +406,40 @@ fn remap_and_write(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Prepares Document for being stored in the document store
|
||||
///
|
||||
/// Method transforms PreTokenizedString values into String
|
||||
/// values.
|
||||
pub fn prepare_doc_for_store(doc: Document, schema: &Schema) -> Document {
|
||||
Document::from(
|
||||
doc.into_iter()
|
||||
.filter(|field_value| schema.get_field_entry(field_value.field()).is_stored())
|
||||
.map(|field_value| match field_value {
|
||||
FieldValue {
|
||||
field,
|
||||
value: Value::PreTokStr(pre_tokenized_text),
|
||||
} => FieldValue {
|
||||
field,
|
||||
value: Value::Str(pre_tokenized_text.text),
|
||||
},
|
||||
field_value => field_value,
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::path::Path;
|
||||
|
||||
use super::compute_initial_table_size;
|
||||
use crate::collector::Count;
|
||||
use crate::directory::RamDirectory;
|
||||
use crate::indexer::json_term_writer::JsonTermWriter;
|
||||
use crate::postings::TermInfo;
|
||||
use crate::query::PhraseQuery;
|
||||
use crate::schema::{IndexRecordOption, Schema, Type, STORED, STRING, TEXT};
|
||||
use crate::store::{Compressor, StoreReader, StoreWriter};
|
||||
use crate::time::format_description::well_known::Rfc3339;
|
||||
use crate::time::OffsetDateTime;
|
||||
use crate::tokenizer::{PreTokenizedString, Token};
|
||||
use crate::{
|
||||
DateTime, Directory, DocAddress, DocSet, Document, Index, Postings, Term, TERMINATED,
|
||||
};
|
||||
use crate::{DateTime, DocAddress, DocSet, Document, Index, Postings, Term, TERMINATED};
|
||||
|
||||
#[test]
|
||||
fn test_hashmap_size() {
|
||||
@@ -489,29 +469,14 @@ mod tests {
|
||||
|
||||
doc.add_pre_tokenized_text(text_field, pre_tokenized_text);
|
||||
doc.add_text(text_field, "title");
|
||||
let prepared_doc = super::prepare_doc_for_store(doc, &schema);
|
||||
|
||||
let path = Path::new("store");
|
||||
let directory = RamDirectory::create();
|
||||
let store_wrt = directory.open_write(path).unwrap();
|
||||
|
||||
let mut store_writer = StoreWriter::new(store_wrt, Compressor::None, 0, false).unwrap();
|
||||
store_writer.store(&doc, &schema).unwrap();
|
||||
store_writer.close().unwrap();
|
||||
|
||||
let reader = StoreReader::open(directory.open_read(path).unwrap(), 0).unwrap();
|
||||
let doc = reader.get(0).unwrap();
|
||||
|
||||
assert_eq!(doc.value_count(), 2);
|
||||
let mut field_value_iter = doc.field_values();
|
||||
assert_eq!(prepared_doc.field_values().len(), 2);
|
||||
assert_eq!(prepared_doc.field_values()[0].value().as_text(), Some("A"));
|
||||
assert_eq!(
|
||||
field_value_iter.next().unwrap().value().as_text(),
|
||||
Some("A")
|
||||
);
|
||||
assert_eq!(
|
||||
field_value_iter.next().unwrap().value().as_text(),
|
||||
prepared_doc.field_values()[1].value().as_text(),
|
||||
Some("title")
|
||||
);
|
||||
assert!(field_value_iter.next().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -561,7 +526,8 @@ mod tests {
|
||||
let inv_idx = segment_reader.inverted_index(json_field).unwrap();
|
||||
let term_dict = inv_idx.terms();
|
||||
|
||||
let mut term = Term::with_type_and_field(Type::Json, json_field);
|
||||
let mut term = Term::new();
|
||||
term.set_field(Type::Json, json_field);
|
||||
let mut term_stream = term_dict.stream().unwrap();
|
||||
|
||||
let mut json_term_writer = JsonTermWriter::wrap(&mut term);
|
||||
@@ -654,7 +620,8 @@ mod tests {
|
||||
let searcher = reader.searcher();
|
||||
let segment_reader = searcher.segment_reader(0u32);
|
||||
let inv_index = segment_reader.inverted_index(json_field).unwrap();
|
||||
let mut term = Term::with_type_and_field(Type::Json, json_field);
|
||||
let mut term = Term::new();
|
||||
term.set_field(Type::Json, json_field);
|
||||
let mut json_term_writer = JsonTermWriter::wrap(&mut term);
|
||||
json_term_writer.push_path_segment("mykey");
|
||||
json_term_writer.set_str("token");
|
||||
@@ -698,7 +665,8 @@ mod tests {
|
||||
let searcher = reader.searcher();
|
||||
let segment_reader = searcher.segment_reader(0u32);
|
||||
let inv_index = segment_reader.inverted_index(json_field).unwrap();
|
||||
let mut term = Term::with_type_and_field(Type::Json, json_field);
|
||||
let mut term = Term::new();
|
||||
term.set_field(Type::Json, json_field);
|
||||
let mut json_term_writer = JsonTermWriter::wrap(&mut term);
|
||||
json_term_writer.push_path_segment("mykey");
|
||||
json_term_writer.set_str("two tokens");
|
||||
@@ -743,7 +711,8 @@ mod tests {
|
||||
writer.commit().unwrap();
|
||||
let reader = index.reader().unwrap();
|
||||
let searcher = reader.searcher();
|
||||
let mut term = Term::with_type_and_field(Type::Json, json_field);
|
||||
let mut term = Term::new();
|
||||
term.set_field(Type::Json, json_field);
|
||||
let mut json_term_writer = JsonTermWriter::wrap(&mut term);
|
||||
json_term_writer.push_path_segment("mykey");
|
||||
json_term_writer.push_path_segment("field");
|
||||
@@ -758,124 +727,4 @@ mod tests {
|
||||
let phrase_query = PhraseQuery::new(vec![nothello_term, happy_term]);
|
||||
assert_eq!(searcher.search(&phrase_query, &Count).unwrap(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bug_regression_1629_position_when_array_with_a_field_value_that_does_not_contain_any_token(
|
||||
) {
|
||||
// We experienced a bug where we would have a position underflow when computing position
|
||||
// delta in an horrible corner case.
|
||||
//
|
||||
// See the commit with this unit test if you want the details.
|
||||
let mut schema_builder = Schema::builder();
|
||||
let text = schema_builder.add_text_field("text", TEXT);
|
||||
let schema = schema_builder.build();
|
||||
let doc = schema
|
||||
.parse_document(r#"{"text": [ "bbb", "aaa", "", "aaa"]}"#)
|
||||
.unwrap();
|
||||
let index = Index::create_in_ram(schema);
|
||||
let mut index_writer = index.writer_for_tests().unwrap();
|
||||
index_writer.add_document(doc).unwrap();
|
||||
// On debug this did panic on the underflow
|
||||
index_writer.commit().unwrap();
|
||||
let reader = index.reader().unwrap();
|
||||
let searcher = reader.searcher();
|
||||
let seg_reader = searcher.segment_reader(0);
|
||||
let inv_index = seg_reader.inverted_index(text).unwrap();
|
||||
let term = Term::from_field_text(text, "aaa");
|
||||
let mut postings = inv_index
|
||||
.read_postings(&term, IndexRecordOption::WithFreqsAndPositions)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(postings.doc(), 0u32);
|
||||
let mut positions = Vec::new();
|
||||
postings.positions(&mut positions);
|
||||
// On release this was [2, 1]. (< note the decreasing values)
|
||||
assert_eq!(positions, &[2, 5]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multiple_field_value_and_long_tokens() {
|
||||
let mut schema_builder = Schema::builder();
|
||||
let text = schema_builder.add_text_field("text", TEXT);
|
||||
let schema = schema_builder.build();
|
||||
let mut doc = Document::default();
|
||||
// This is a bit of a contrived example.
|
||||
let tokens = PreTokenizedString {
|
||||
text: "roller-coaster".to_string(),
|
||||
tokens: vec![Token {
|
||||
offset_from: 0,
|
||||
offset_to: 14,
|
||||
position: 0,
|
||||
text: "rollercoaster".to_string(),
|
||||
position_length: 2,
|
||||
}],
|
||||
};
|
||||
doc.add_pre_tokenized_text(text, tokens.clone());
|
||||
doc.add_pre_tokenized_text(text, tokens);
|
||||
let index = Index::create_in_ram(schema);
|
||||
let mut index_writer = index.writer_for_tests().unwrap();
|
||||
index_writer.add_document(doc).unwrap();
|
||||
index_writer.commit().unwrap();
|
||||
let reader = index.reader().unwrap();
|
||||
let searcher = reader.searcher();
|
||||
let seg_reader = searcher.segment_reader(0);
|
||||
let inv_index = seg_reader.inverted_index(text).unwrap();
|
||||
let term = Term::from_field_text(text, "rollercoaster");
|
||||
let mut postings = inv_index
|
||||
.read_postings(&term, IndexRecordOption::WithFreqsAndPositions)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(postings.doc(), 0u32);
|
||||
let mut positions = Vec::new();
|
||||
postings.positions(&mut positions);
|
||||
assert_eq!(positions, &[0, 3]); //< as opposed to 0, 2 if we had a position length of 1.
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_last_token_not_ending_last() {
|
||||
let mut schema_builder = Schema::builder();
|
||||
let text = schema_builder.add_text_field("text", TEXT);
|
||||
let schema = schema_builder.build();
|
||||
let mut doc = Document::default();
|
||||
// This is a bit of a contrived example.
|
||||
let tokens = PreTokenizedString {
|
||||
text: "contrived-example".to_string(), //< I can't think of a use case where this corner case happens in real life.
|
||||
tokens: vec![
|
||||
Token {
|
||||
// Not the last token, yet ends after the last token.
|
||||
offset_from: 0,
|
||||
offset_to: 14,
|
||||
position: 0,
|
||||
text: "long_token".to_string(),
|
||||
position_length: 3,
|
||||
},
|
||||
Token {
|
||||
offset_from: 0,
|
||||
offset_to: 14,
|
||||
position: 1,
|
||||
text: "short".to_string(),
|
||||
position_length: 1,
|
||||
},
|
||||
],
|
||||
};
|
||||
doc.add_pre_tokenized_text(text, tokens);
|
||||
doc.add_text(text, "hello");
|
||||
let index = Index::create_in_ram(schema);
|
||||
let mut index_writer = index.writer_for_tests().unwrap();
|
||||
index_writer.add_document(doc).unwrap();
|
||||
index_writer.commit().unwrap();
|
||||
let reader = index.reader().unwrap();
|
||||
let searcher = reader.searcher();
|
||||
let seg_reader = searcher.segment_reader(0);
|
||||
let inv_index = seg_reader.inverted_index(text).unwrap();
|
||||
let term = Term::from_field_text(text, "hello");
|
||||
let mut postings = inv_index
|
||||
.read_postings(&term, IndexRecordOption::WithFreqsAndPositions)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(postings.doc(), 0u32);
|
||||
let mut positions = Vec::new();
|
||||
postings.positions(&mut positions);
|
||||
assert_eq!(positions, &[4]); //< as opposed to 3 if we had a position length of 1.
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ pub(crate) struct RemappedDocIdColumn<'a> {
|
||||
fast_field_readers: Vec<Arc<dyn Column<u64>>>,
|
||||
min_value: u64,
|
||||
max_value: u64,
|
||||
num_vals: u32,
|
||||
num_vals: u64,
|
||||
}
|
||||
|
||||
fn compute_min_max_val(
|
||||
@@ -32,7 +32,7 @@ fn compute_min_max_val(
|
||||
// we need to recompute the max / min
|
||||
segment_reader
|
||||
.doc_ids_alive()
|
||||
.map(|doc_id| u64_reader.get_val(doc_id))
|
||||
.map(|doc_id| u64_reader.get_val(doc_id as u64))
|
||||
.minmax()
|
||||
.into_option()
|
||||
}
|
||||
@@ -73,13 +73,13 @@ impl<'a> RemappedDocIdColumn<'a> {
|
||||
fast_field_readers,
|
||||
min_value,
|
||||
max_value,
|
||||
num_vals: doc_id_mapping.len() as u32,
|
||||
num_vals: doc_id_mapping.len() as u64,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Column for RemappedDocIdColumn<'a> {
|
||||
fn get_val(&self, _doc: u32) -> u64 {
|
||||
fn get_val(&self, _doc: u64) -> u64 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
@@ -90,7 +90,7 @@ impl<'a> Column for RemappedDocIdColumn<'a> {
|
||||
.map(|old_doc_addr| {
|
||||
let fast_field_reader =
|
||||
&self.fast_field_readers[old_doc_addr.segment_ord as usize];
|
||||
fast_field_reader.get_val(old_doc_addr.doc_id)
|
||||
fast_field_reader.get_val(old_doc_addr.doc_id as u64)
|
||||
}),
|
||||
)
|
||||
}
|
||||
@@ -102,7 +102,7 @@ impl<'a> Column for RemappedDocIdColumn<'a> {
|
||||
self.max_value
|
||||
}
|
||||
|
||||
fn num_vals(&self) -> u32 {
|
||||
fn num_vals(&self) -> u64 {
|
||||
self.num_vals
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,18 +2,17 @@ use std::cmp;
|
||||
|
||||
use fastfield_codecs::Column;
|
||||
|
||||
use super::flat_map_with_buffer::FlatMapWithBufferIter;
|
||||
use crate::fastfield::{MultiValueLength, MultiValuedFastFieldReader};
|
||||
use crate::fastfield::MultiValuedFastFieldReader;
|
||||
use crate::indexer::doc_id_mapping::SegmentDocIdMapping;
|
||||
use crate::schema::Field;
|
||||
use crate::{DocAddress, SegmentReader};
|
||||
use crate::SegmentReader;
|
||||
|
||||
pub(crate) struct RemappedDocIdMultiValueColumn<'a> {
|
||||
doc_id_mapping: &'a SegmentDocIdMapping,
|
||||
fast_field_readers: Vec<MultiValuedFastFieldReader<u64>>,
|
||||
min_value: u64,
|
||||
max_value: u64,
|
||||
num_vals: u32,
|
||||
num_vals: u64,
|
||||
}
|
||||
|
||||
impl<'a> RemappedDocIdMultiValueColumn<'a> {
|
||||
@@ -61,13 +60,13 @@ impl<'a> RemappedDocIdMultiValueColumn<'a> {
|
||||
fast_field_readers,
|
||||
min_value,
|
||||
max_value,
|
||||
num_vals: num_vals as u32,
|
||||
num_vals: num_vals as u64,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Column for RemappedDocIdMultiValueColumn<'a> {
|
||||
fn get_val(&self, _pos: u32) -> u64 {
|
||||
fn get_val(&self, _pos: u64) -> u64 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
@@ -75,9 +74,11 @@ impl<'a> Column for RemappedDocIdMultiValueColumn<'a> {
|
||||
Box::new(
|
||||
self.doc_id_mapping
|
||||
.iter_old_doc_addrs()
|
||||
.flat_map_with_buffer(|old_doc_addr: DocAddress, buffer| {
|
||||
.flat_map(|old_doc_addr| {
|
||||
let ff_reader = &self.fast_field_readers[old_doc_addr.segment_ord as usize];
|
||||
ff_reader.get_vals(old_doc_addr.doc_id, buffer);
|
||||
let mut vals = Vec::new();
|
||||
ff_reader.get_vals(old_doc_addr.doc_id, &mut vals);
|
||||
vals.into_iter()
|
||||
}),
|
||||
)
|
||||
}
|
||||
@@ -89,80 +90,7 @@ impl<'a> Column for RemappedDocIdMultiValueColumn<'a> {
|
||||
self.max_value
|
||||
}
|
||||
|
||||
fn num_vals(&self) -> u32 {
|
||||
self.num_vals
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct RemappedDocIdMultiValueIndexColumn<'a, T: MultiValueLength> {
|
||||
doc_id_mapping: &'a SegmentDocIdMapping,
|
||||
multi_value_length_readers: Vec<&'a T>,
|
||||
min_value: u64,
|
||||
max_value: u64,
|
||||
num_vals: u32,
|
||||
}
|
||||
|
||||
impl<'a, T: MultiValueLength> RemappedDocIdMultiValueIndexColumn<'a, T> {
|
||||
pub(crate) fn new(
|
||||
segment_and_ff_readers: &'a [(&'a SegmentReader, T)],
|
||||
doc_id_mapping: &'a SegmentDocIdMapping,
|
||||
) -> Self {
|
||||
// We go through a complete first pass to compute the minimum and the
|
||||
// maximum value and initialize our Column.
|
||||
let mut num_vals = 0;
|
||||
let min_value = 0;
|
||||
let mut max_value = 0;
|
||||
let mut multi_value_length_readers = Vec::with_capacity(segment_and_ff_readers.len());
|
||||
for segment_and_ff_reader in segment_and_ff_readers {
|
||||
let segment_reader = segment_and_ff_reader.0;
|
||||
let multi_value_length_reader = &segment_and_ff_reader.1;
|
||||
if !segment_reader.has_deletes() {
|
||||
max_value += multi_value_length_reader.get_total_len();
|
||||
} else {
|
||||
for doc in segment_reader.doc_ids_alive() {
|
||||
max_value += multi_value_length_reader.get_len(doc);
|
||||
}
|
||||
}
|
||||
num_vals += segment_reader.num_docs();
|
||||
multi_value_length_readers.push(multi_value_length_reader);
|
||||
}
|
||||
Self {
|
||||
doc_id_mapping,
|
||||
multi_value_length_readers,
|
||||
min_value,
|
||||
max_value,
|
||||
num_vals,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, T: MultiValueLength + Send + Sync> Column for RemappedDocIdMultiValueIndexColumn<'a, T> {
|
||||
fn get_val(&self, _pos: u32) -> u64 {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn iter(&self) -> Box<dyn Iterator<Item = u64> + '_> {
|
||||
let mut offset = 0;
|
||||
Box::new(
|
||||
std::iter::once(0).chain(self.doc_id_mapping.iter_old_doc_addrs().map(
|
||||
move |old_doc_addr| {
|
||||
let ff_reader =
|
||||
&self.multi_value_length_readers[old_doc_addr.segment_ord as usize];
|
||||
offset += ff_reader.get_len(old_doc_addr.doc_id);
|
||||
offset
|
||||
},
|
||||
)),
|
||||
)
|
||||
}
|
||||
fn min_value(&self) -> u64 {
|
||||
self.min_value
|
||||
}
|
||||
|
||||
fn max_value(&self) -> u64 {
|
||||
self.max_value
|
||||
}
|
||||
|
||||
fn num_vals(&self) -> u32 {
|
||||
fn num_vals(&self) -> u64 {
|
||||
self.num_vals
|
||||
}
|
||||
}
|
||||
|
||||
@@ -311,7 +311,7 @@ pub use crate::postings::Postings;
|
||||
pub use crate::schema::{DateOptions, DatePrecision, Document, Term};
|
||||
|
||||
/// Index format version.
|
||||
const INDEX_FORMAT_VERSION: u32 = 5;
|
||||
const INDEX_FORMAT_VERSION: u32 = 4;
|
||||
|
||||
/// Structure version for the index.
|
||||
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
@@ -819,7 +819,7 @@ pub mod tests {
|
||||
fn test_indexedfield_not_in_documents() -> crate::Result<()> {
|
||||
let mut schema_builder = Schema::builder();
|
||||
let text_field = schema_builder.add_text_field("text", TEXT);
|
||||
let absent_field = schema_builder.add_text_field("absent_text", TEXT);
|
||||
let absent_field = schema_builder.add_text_field("text", TEXT);
|
||||
let schema = schema_builder.build();
|
||||
let index = Index::create_in_ram(schema);
|
||||
let mut index_writer = index.writer_for_tests()?;
|
||||
@@ -1001,7 +1001,7 @@ pub mod tests {
|
||||
let fast_field_signed = schema_builder.add_i64_field("signed", FAST);
|
||||
let fast_field_float = schema_builder.add_f64_field("float", FAST);
|
||||
let text_field = schema_builder.add_text_field("text", TEXT);
|
||||
let stored_int_field = schema_builder.add_u64_field("stored_int", STORED);
|
||||
let stored_int_field = schema_builder.add_u64_field("text", STORED);
|
||||
let schema = schema_builder.build();
|
||||
|
||||
let index = Index::create_in_ram(schema);
|
||||
|
||||
@@ -3,7 +3,7 @@ use std::io;
|
||||
use crate::fastfield::MultiValuedFastFieldWriter;
|
||||
use crate::indexer::doc_id_mapping::DocIdMapping;
|
||||
use crate::postings::postings_writer::SpecializedPostingsWriter;
|
||||
use crate::postings::recorder::{BufferLender, DocIdRecorder, Recorder};
|
||||
use crate::postings::recorder::{BufferLender, NothingRecorder, Recorder};
|
||||
use crate::postings::stacker::Addr;
|
||||
use crate::postings::{
|
||||
FieldSerializer, IndexingContext, IndexingPosition, PostingsWriter, UnorderedTermId,
|
||||
@@ -16,7 +16,7 @@ use crate::{DocId, Term};
|
||||
#[derive(Default)]
|
||||
pub(crate) struct JsonPostingsWriter<Rec: Recorder> {
|
||||
str_posting_writer: SpecializedPostingsWriter<Rec>,
|
||||
non_str_posting_writer: SpecializedPostingsWriter<DocIdRecorder>,
|
||||
non_str_posting_writer: SpecializedPostingsWriter<NothingRecorder>,
|
||||
}
|
||||
|
||||
impl<Rec: Recorder> From<JsonPostingsWriter<Rec>> for Box<dyn PostingsWriter> {
|
||||
@@ -77,7 +77,7 @@ impl<Rec: Recorder> PostingsWriter for JsonPostingsWriter<Rec> {
|
||||
serializer,
|
||||
)?;
|
||||
} else {
|
||||
SpecializedPostingsWriter::<DocIdRecorder>::serialize_one_term(
|
||||
SpecializedPostingsWriter::<NothingRecorder>::serialize_one_term(
|
||||
term,
|
||||
*addr,
|
||||
doc_id_map,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use crate::postings::json_postings_writer::JsonPostingsWriter;
|
||||
use crate::postings::postings_writer::SpecializedPostingsWriter;
|
||||
use crate::postings::recorder::{DocIdRecorder, TermFrequencyRecorder, TfAndPositionRecorder};
|
||||
use crate::postings::recorder::{NothingRecorder, TermFrequencyRecorder, TfAndPositionRecorder};
|
||||
use crate::postings::PostingsWriter;
|
||||
use crate::schema::{Field, FieldEntry, FieldType, IndexRecordOption, Schema};
|
||||
|
||||
@@ -34,7 +34,7 @@ fn posting_writer_from_field_entry(field_entry: &FieldEntry) -> Box<dyn Postings
|
||||
.get_indexing_options()
|
||||
.map(|indexing_options| match indexing_options.index_option() {
|
||||
IndexRecordOption::Basic => {
|
||||
SpecializedPostingsWriter::<DocIdRecorder>::default().into()
|
||||
SpecializedPostingsWriter::<NothingRecorder>::default().into()
|
||||
}
|
||||
IndexRecordOption::WithFreqs => {
|
||||
SpecializedPostingsWriter::<TermFrequencyRecorder>::default().into()
|
||||
@@ -43,20 +43,19 @@ fn posting_writer_from_field_entry(field_entry: &FieldEntry) -> Box<dyn Postings
|
||||
SpecializedPostingsWriter::<TfAndPositionRecorder>::default().into()
|
||||
}
|
||||
})
|
||||
.unwrap_or_else(|| SpecializedPostingsWriter::<DocIdRecorder>::default().into()),
|
||||
.unwrap_or_else(|| SpecializedPostingsWriter::<NothingRecorder>::default().into()),
|
||||
FieldType::U64(_)
|
||||
| FieldType::I64(_)
|
||||
| FieldType::F64(_)
|
||||
| FieldType::Bool(_)
|
||||
| FieldType::Date(_)
|
||||
| FieldType::Bytes(_)
|
||||
| FieldType::IpAddr(_)
|
||||
| FieldType::Facet(_) => Box::new(SpecializedPostingsWriter::<DocIdRecorder>::default()),
|
||||
| FieldType::Facet(_) => Box::new(SpecializedPostingsWriter::<NothingRecorder>::default()),
|
||||
FieldType::JsonObject(ref json_object_options) => {
|
||||
if let Some(text_indexing_option) = json_object_options.get_text_indexing_options() {
|
||||
match text_indexing_option.index_option() {
|
||||
IndexRecordOption::Basic => {
|
||||
JsonPostingsWriter::<DocIdRecorder>::default().into()
|
||||
JsonPostingsWriter::<NothingRecorder>::default().into()
|
||||
}
|
||||
IndexRecordOption::WithFreqs => {
|
||||
JsonPostingsWriter::<TermFrequencyRecorder>::default().into()
|
||||
@@ -66,7 +65,7 @@ fn posting_writer_from_field_entry(field_entry: &FieldEntry) -> Box<dyn Postings
|
||||
}
|
||||
}
|
||||
} else {
|
||||
JsonPostingsWriter::<DocIdRecorder>::default().into()
|
||||
JsonPostingsWriter::<NothingRecorder>::default().into()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ use std::io;
|
||||
use std::marker::PhantomData;
|
||||
use std::ops::Range;
|
||||
|
||||
use rustc_hash::FxHashMap;
|
||||
use fnv::FnvHashMap;
|
||||
|
||||
use super::stacker::Addr;
|
||||
use crate::fastfield::MultiValuedFastFieldWriter;
|
||||
@@ -56,12 +56,12 @@ pub(crate) fn serialize_postings(
|
||||
doc_id_map: Option<&DocIdMapping>,
|
||||
schema: &Schema,
|
||||
serializer: &mut InvertedIndexSerializer,
|
||||
) -> crate::Result<HashMap<Field, FxHashMap<UnorderedTermId, TermOrdinal>>> {
|
||||
) -> crate::Result<HashMap<Field, FnvHashMap<UnorderedTermId, TermOrdinal>>> {
|
||||
let mut term_offsets: Vec<(Term<&[u8]>, Addr, UnorderedTermId)> =
|
||||
Vec::with_capacity(ctx.term_index.len());
|
||||
term_offsets.extend(ctx.term_index.iter());
|
||||
term_offsets.sort_unstable_by_key(|(k, _, _)| k.clone());
|
||||
let mut unordered_term_mappings: HashMap<Field, FxHashMap<UnorderedTermId, TermOrdinal>> =
|
||||
let mut unordered_term_mappings: HashMap<Field, FnvHashMap<UnorderedTermId, TermOrdinal>> =
|
||||
HashMap::new();
|
||||
|
||||
let field_offsets = make_field_partition(&term_offsets);
|
||||
@@ -74,7 +74,7 @@ pub(crate) fn serialize_postings(
|
||||
let unordered_term_ids = term_offsets[byte_offsets.clone()]
|
||||
.iter()
|
||||
.map(|&(_, _, bucket)| bucket);
|
||||
let mapping: FxHashMap<UnorderedTermId, TermOrdinal> = unordered_term_ids
|
||||
let mapping: FnvHashMap<UnorderedTermId, TermOrdinal> = unordered_term_ids
|
||||
.enumerate()
|
||||
.map(|(term_ord, unord_term_id)| {
|
||||
(unord_term_id as UnorderedTermId, term_ord as TermOrdinal)
|
||||
@@ -89,7 +89,6 @@ pub(crate) fn serialize_postings(
|
||||
| FieldType::Bool(_) => {}
|
||||
FieldType::Bytes(_) => {}
|
||||
FieldType::JsonObject(_) => {}
|
||||
FieldType::IpAddr(_) => {}
|
||||
}
|
||||
|
||||
let postings_writer = per_field_postings_writers.get_for_field(field);
|
||||
@@ -153,9 +152,9 @@ pub(crate) trait PostingsWriter: Send + Sync {
|
||||
indexing_position: &mut IndexingPosition,
|
||||
mut term_id_fast_field_writer_opt: Option<&mut MultiValuedFastFieldWriter>,
|
||||
) {
|
||||
let end_of_path_idx = term_buffer.len_bytes();
|
||||
let end_of_path_idx = term_buffer.as_slice().len();
|
||||
let mut num_tokens = 0;
|
||||
let mut end_position = indexing_position.end_position;
|
||||
let mut end_position = 0;
|
||||
token_stream.process(&mut |token: &Token| {
|
||||
// We skip all tokens with a len greater than u16.
|
||||
if token.text.len() > MAX_TOKEN_LEN {
|
||||
@@ -167,10 +166,10 @@ pub(crate) trait PostingsWriter: Send + Sync {
|
||||
);
|
||||
return;
|
||||
}
|
||||
term_buffer.truncate_value_bytes(end_of_path_idx);
|
||||
term_buffer.truncate(end_of_path_idx);
|
||||
term_buffer.append_bytes(token.text.as_bytes());
|
||||
let start_position = indexing_position.end_position + token.position as u32;
|
||||
end_position = end_position.max(start_position + token.position_length as u32);
|
||||
end_position = start_position + token.position_length as u32;
|
||||
let unordered_term_id = self.subscribe(doc_id, start_position, term_buffer, ctx);
|
||||
if let Some(term_id_fast_field_writer) = term_id_fast_field_writer_opt.as_mut() {
|
||||
term_id_fast_field_writer.add_val(unordered_term_id);
|
||||
@@ -181,7 +180,7 @@ pub(crate) trait PostingsWriter: Send + Sync {
|
||||
|
||||
indexing_position.end_position = end_position + POSITION_GAP;
|
||||
indexing_position.num_tokens += num_tokens;
|
||||
term_buffer.truncate_value_bytes(end_of_path_idx);
|
||||
term_buffer.truncate(end_of_path_idx);
|
||||
}
|
||||
|
||||
fn total_num_tokens(&self) -> u64;
|
||||
|
||||
@@ -47,11 +47,11 @@ impl<'a> Iterator for VInt32Reader<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
/// `Recorder` is in charge of recording relevant information about
|
||||
/// Recorder is in charge of recording relevant information about
|
||||
/// the presence of a term in a document.
|
||||
///
|
||||
/// Depending on the [`TextOptions`](crate::schema::TextOptions) associated
|
||||
/// with the field, the recorder may record:
|
||||
/// Depending on the `TextIndexingOptions` associated with the
|
||||
/// field, the recorder may records
|
||||
/// * the document frequency
|
||||
/// * the document id
|
||||
/// * the term frequency
|
||||
@@ -83,21 +83,21 @@ pub(crate) trait Recorder: Copy + Default + Send + Sync + 'static {
|
||||
|
||||
/// Only records the doc ids
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct DocIdRecorder {
|
||||
pub struct NothingRecorder {
|
||||
stack: ExpUnrolledLinkedList,
|
||||
current_doc: DocId,
|
||||
}
|
||||
|
||||
impl Default for DocIdRecorder {
|
||||
impl Default for NothingRecorder {
|
||||
fn default() -> Self {
|
||||
DocIdRecorder {
|
||||
NothingRecorder {
|
||||
stack: ExpUnrolledLinkedList::new(),
|
||||
current_doc: u32::MAX,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Recorder for DocIdRecorder {
|
||||
impl Recorder for NothingRecorder {
|
||||
fn current_doc(&self) -> DocId {
|
||||
self.current_doc
|
||||
}
|
||||
|
||||
@@ -98,7 +98,7 @@ impl<'a> Iterator for Iter<'a> {
|
||||
/// # Panics if n == 0
|
||||
fn compute_previous_power_of_two(n: usize) -> usize {
|
||||
assert!(n > 0);
|
||||
let msb = (63u32 - (n as u64).leading_zeros()) as u8;
|
||||
let msb = (63u32 - n.leading_zeros()) as u8;
|
||||
1 << msb
|
||||
}
|
||||
|
||||
@@ -199,7 +199,7 @@ impl TermHashMap {
|
||||
/// `update` create a new entry for a given key if it does not exists
|
||||
/// or updates the existing entry.
|
||||
///
|
||||
/// The actual logic for this update is define in the `updater`
|
||||
/// The actual logic for this update is define in the the `updater`
|
||||
/// argument.
|
||||
///
|
||||
/// If the key is not present, `updater` will receive `None` and
|
||||
|
||||
@@ -86,7 +86,10 @@ impl DocSet for BitSetDocSet {
|
||||
self.doc
|
||||
}
|
||||
|
||||
/// Returns the number of values set in the underlying bitset.
|
||||
/// Returns half of the `max_doc`
|
||||
/// This is quite a terrible heuristic,
|
||||
/// but we don't have access to any better
|
||||
/// value.
|
||||
fn size_hint(&self) -> u32 {
|
||||
self.docs.len() as u32
|
||||
}
|
||||
|
||||
@@ -212,12 +212,12 @@ pub fn block_wand(
|
||||
}
|
||||
|
||||
/// Specialized version of [`block_wand`] for a single scorer.
|
||||
/// In this case, the algorithm is simple, readable and faster (~ x3)
|
||||
/// In this case, the algorithm is simple and readable and faster (~ x3)
|
||||
/// than the generic algorithm.
|
||||
/// The algorithm behaves as follows:
|
||||
/// - While we don't hit the end of the docset:
|
||||
/// - While the block max score is under the `threshold`, go to the next block.
|
||||
/// - On a block, advance until the end and execute `callback` when the doc score is greater or
|
||||
/// - On a block, advance until the end and execute `callback`` when the doc score is greater or
|
||||
/// equal to the `threshold`.
|
||||
pub fn block_wand_single_scorer(
|
||||
mut scorer: TermScorer,
|
||||
|
||||
@@ -18,7 +18,6 @@ mod phrase_query;
|
||||
mod query;
|
||||
mod query_parser;
|
||||
mod range_query;
|
||||
mod range_query_ip_fastfield;
|
||||
mod regex_query;
|
||||
mod reqopt_scorer;
|
||||
mod scorer;
|
||||
|
||||
@@ -31,7 +31,7 @@ pub struct MoreLikeThisQuery {
|
||||
#[derive(Debug, PartialEq, Clone)]
|
||||
enum TargetDocument {
|
||||
DocumentAdress(DocAddress),
|
||||
DocumentFields(Vec<(Field, Vec<Value<'static>>)>),
|
||||
DocumentFields(Vec<(Field, Vec<Value>)>),
|
||||
}
|
||||
|
||||
impl MoreLikeThisQuery {
|
||||
@@ -160,10 +160,7 @@ impl MoreLikeThisQueryBuilder {
|
||||
/// that will be used to compose the resulting query.
|
||||
/// This interface is meant to be used when you want to provide your own set of fields
|
||||
/// not necessarily from a specific document.
|
||||
pub fn with_document_fields(
|
||||
self,
|
||||
doc_fields: Vec<(Field, Vec<Value<'static>>)>,
|
||||
) -> MoreLikeThisQuery {
|
||||
pub fn with_document_fields(self, doc_fields: Vec<(Field, Vec<Value>)>) -> MoreLikeThisQuery {
|
||||
MoreLikeThisQuery {
|
||||
mlt: self.mlt,
|
||||
target: TargetDocument::DocumentFields(doc_fields),
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
use std::collections::HashMap;
|
||||
use std::net::{AddrParseError, IpAddr};
|
||||
use std::num::{ParseFloatError, ParseIntError};
|
||||
use std::ops::Bound;
|
||||
use std::str::{FromStr, ParseBoolError};
|
||||
@@ -16,7 +15,7 @@ use crate::query::{
|
||||
TermQuery,
|
||||
};
|
||||
use crate::schema::{
|
||||
Facet, FacetParseError, Field, FieldType, IndexRecordOption, IntoIpv6Addr, Schema, Term, Type,
|
||||
Facet, FacetParseError, Field, FieldType, IndexRecordOption, Schema, Term, Type,
|
||||
};
|
||||
use crate::time::format_description::well_known::Rfc3339;
|
||||
use crate::time::OffsetDateTime;
|
||||
@@ -85,9 +84,6 @@ pub enum QueryParserError {
|
||||
/// The format for the facet field is invalid.
|
||||
#[error("The facet field is malformed: {0}")]
|
||||
FacetFormatError(#[from] FacetParseError),
|
||||
/// The format for the ip field is invalid.
|
||||
#[error("The ip field is malformed: {0}")]
|
||||
IpFormatError(#[from] AddrParseError),
|
||||
}
|
||||
|
||||
/// Recursively remove empty clause from the AST
|
||||
@@ -404,10 +400,6 @@ impl QueryParser {
|
||||
let bytes = base64::decode(phrase).map_err(QueryParserError::ExpectedBase64)?;
|
||||
Ok(Term::from_field_bytes(field, &bytes))
|
||||
}
|
||||
FieldType::IpAddr(_) => {
|
||||
let ip_v6 = IpAddr::from_str(phrase)?.into_ipv6_addr();
|
||||
Ok(Term::from_field_ip_addr(field, ip_v6))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -514,11 +506,6 @@ impl QueryParser {
|
||||
let bytes_term = Term::from_field_bytes(field, &bytes);
|
||||
Ok(vec![LogicalLiteral::Term(bytes_term)])
|
||||
}
|
||||
FieldType::IpAddr(_) => {
|
||||
let ip_v6 = IpAddr::from_str(phrase)?.into_ipv6_addr();
|
||||
let term = Term::from_field_ip_addr(field, ip_v6);
|
||||
Ok(vec![LogicalLiteral::Term(term)])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -743,7 +730,7 @@ fn generate_literals_for_json_object(
|
||||
index_record_option: IndexRecordOption,
|
||||
) -> Result<Vec<LogicalLiteral>, QueryParserError> {
|
||||
let mut logical_literals = Vec::new();
|
||||
let mut term = Term::with_capacity(100);
|
||||
let mut term = Term::new();
|
||||
let mut json_term_writer =
|
||||
JsonTermWriter::from_field_and_json_path(field, json_path, &mut term);
|
||||
if let Some(term) = convert_to_fast_value_and_get_term(&mut json_term_writer, phrase) {
|
||||
|
||||
@@ -6,13 +6,12 @@ use common::BitSet;
|
||||
use crate::core::{Searcher, SegmentReader};
|
||||
use crate::error::TantivyError;
|
||||
use crate::query::explanation::does_not_match;
|
||||
use crate::query::range_query_ip_fastfield::IPFastFieldRangeWeight;
|
||||
use crate::query::{BitSetDocSet, ConstScorer, Explanation, Query, Scorer, Weight};
|
||||
use crate::schema::{Field, IndexRecordOption, Term, Type};
|
||||
use crate::termdict::{TermDictionary, TermStreamer};
|
||||
use crate::{DocId, Score};
|
||||
|
||||
pub(crate) fn map_bound<TFrom, TTo, Transform: Fn(&TFrom) -> TTo>(
|
||||
fn map_bound<TFrom, TTo, Transform: Fn(&TFrom) -> TTo>(
|
||||
bound: &Bound<TFrom>,
|
||||
transform: &Transform,
|
||||
) -> Bound<TTo> {
|
||||
@@ -30,17 +29,8 @@ pub(crate) fn map_bound<TFrom, TTo, Transform: Fn(&TFrom) -> TTo>(
|
||||
///
|
||||
/// # Implementation
|
||||
///
|
||||
/// ## Default
|
||||
/// The default implementation collects all documents _upfront_ into a `BitSet`.
|
||||
/// This is done by iterating over the terms within the range and loading all docs for each
|
||||
/// `TermInfo` from the inverted index (posting list) and put them into a `BitSet`.
|
||||
/// Depending on the number of terms matched, this is a potentially expensive operation.
|
||||
///
|
||||
/// ## IP fast field
|
||||
/// For IP fast fields a custom variant is used, by scanning the fast field. Unlike the default
|
||||
/// variant we can walk in a lazy fashion over it, since the fastfield is implicit orderered by
|
||||
/// DocId.
|
||||
///
|
||||
/// The current implement will iterate over the terms within the range
|
||||
/// and append all of the document cross into a `BitSet`.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
@@ -259,8 +249,7 @@ impl Query for RangeQuery {
|
||||
_scoring_enabled: bool,
|
||||
) -> crate::Result<Box<dyn Weight>> {
|
||||
let schema = searcher.schema();
|
||||
let field_type = schema.get_field_entry(self.field).field_type();
|
||||
let value_type = field_type.value_type();
|
||||
let value_type = schema.get_field_entry(self.field).field_type().value_type();
|
||||
if value_type != self.value_type {
|
||||
let err_msg = format!(
|
||||
"Create a range query of the type {:?}, when the field given was of type {:?}",
|
||||
@@ -268,20 +257,11 @@ impl Query for RangeQuery {
|
||||
);
|
||||
return Err(TantivyError::SchemaError(err_msg));
|
||||
}
|
||||
|
||||
if field_type.is_ip_addr() && field_type.is_fast() {
|
||||
Ok(Box::new(IPFastFieldRangeWeight::new(
|
||||
self.field,
|
||||
&self.left_bound,
|
||||
&self.right_bound,
|
||||
)))
|
||||
} else {
|
||||
Ok(Box::new(RangeWeight {
|
||||
field: self.field,
|
||||
left_bound: self.left_bound.clone(),
|
||||
right_bound: self.right_bound.clone(),
|
||||
}))
|
||||
}
|
||||
Ok(Box::new(RangeWeight {
|
||||
field: self.field,
|
||||
left_bound: self.left_bound.clone(),
|
||||
right_bound: self.right_bound.clone(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -348,15 +328,13 @@ impl Weight for RangeWeight {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
use std::net::IpAddr;
|
||||
use std::ops::Bound;
|
||||
use std::str::FromStr;
|
||||
|
||||
use super::RangeQuery;
|
||||
use crate::collector::{Count, TopDocs};
|
||||
use crate::query::QueryParser;
|
||||
use crate::schema::{Document, Field, IntoIpv6Addr, Schema, FAST, INDEXED, STORED, TEXT};
|
||||
use crate::{doc, Index};
|
||||
use crate::schema::{Document, Field, Schema, INDEXED, TEXT};
|
||||
use crate::Index;
|
||||
|
||||
#[test]
|
||||
fn test_range_query_simple() -> crate::Result<()> {
|
||||
@@ -528,165 +506,4 @@ mod tests {
|
||||
assert_eq!(top_docs.len(), 1);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_ip_range_test_posting_list() {
|
||||
search_ip_range_test_opt(false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_ip_range_test() {
|
||||
search_ip_range_test_opt(true);
|
||||
}
|
||||
|
||||
fn search_ip_range_test_opt(with_fast_field: bool) {
|
||||
let mut schema_builder = Schema::builder();
|
||||
let ip_field = if with_fast_field {
|
||||
schema_builder.add_ip_addr_field("ip", INDEXED | STORED | FAST)
|
||||
} else {
|
||||
schema_builder.add_ip_addr_field("ip", INDEXED | STORED)
|
||||
};
|
||||
let text_field = schema_builder.add_text_field("text", TEXT | STORED);
|
||||
let schema = schema_builder.build();
|
||||
let index = Index::create_in_ram(schema);
|
||||
let ip_addr_1 = IpAddr::from_str("127.0.0.10").unwrap().into_ipv6_addr();
|
||||
let ip_addr_2 = IpAddr::from_str("127.0.0.20").unwrap().into_ipv6_addr();
|
||||
|
||||
{
|
||||
let mut index_writer = index.writer(3_000_000).unwrap();
|
||||
for _ in 0..1_000 {
|
||||
index_writer
|
||||
.add_document(doc!(
|
||||
ip_field => ip_addr_1,
|
||||
text_field => "BLUBBER"
|
||||
))
|
||||
.unwrap();
|
||||
}
|
||||
for _ in 0..1_000 {
|
||||
index_writer
|
||||
.add_document(doc!(
|
||||
ip_field => ip_addr_2,
|
||||
text_field => "BLOBBER"
|
||||
))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
index_writer.commit().unwrap();
|
||||
}
|
||||
let reader = index.reader().unwrap();
|
||||
let searcher = reader.searcher();
|
||||
|
||||
let get_num_hits = |query| {
|
||||
let (_top_docs, count) = searcher
|
||||
.search(&query, &(TopDocs::with_limit(10), Count))
|
||||
.unwrap();
|
||||
count
|
||||
};
|
||||
let query_from_text = |text: &str| {
|
||||
QueryParser::for_index(&index, vec![])
|
||||
.parse_query(text)
|
||||
.unwrap()
|
||||
};
|
||||
|
||||
// Inclusive range
|
||||
assert_eq!(
|
||||
get_num_hits(query_from_text("ip:[127.0.0.1 TO 127.0.0.20]")),
|
||||
2000
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
get_num_hits(query_from_text("ip:[127.0.0.10 TO 127.0.0.20]")),
|
||||
2000
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
get_num_hits(query_from_text("ip:[127.0.0.11 TO 127.0.0.20]")),
|
||||
1000
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
get_num_hits(query_from_text("ip:[127.0.0.11 TO 127.0.0.19]")),
|
||||
0
|
||||
);
|
||||
|
||||
assert_eq!(get_num_hits(query_from_text("ip:[127.0.0.11 TO *]")), 1000);
|
||||
assert_eq!(get_num_hits(query_from_text("ip:[127.0.0.21 TO *]")), 0);
|
||||
assert_eq!(get_num_hits(query_from_text("ip:[* TO 127.0.0.9]")), 0);
|
||||
assert_eq!(get_num_hits(query_from_text("ip:[* TO 127.0.0.10]")), 1000);
|
||||
|
||||
// Exclusive range
|
||||
assert_eq!(
|
||||
get_num_hits(query_from_text("ip:{127.0.0.1 TO 127.0.0.20}")),
|
||||
1000
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
get_num_hits(query_from_text("ip:{127.0.0.1 TO 127.0.0.21}")),
|
||||
2000
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
get_num_hits(query_from_text("ip:{127.0.0.10 TO 127.0.0.20}")),
|
||||
0
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
get_num_hits(query_from_text("ip:{127.0.0.11 TO 127.0.0.20}")),
|
||||
0
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
get_num_hits(query_from_text("ip:{127.0.0.11 TO 127.0.0.19}")),
|
||||
0
|
||||
);
|
||||
|
||||
assert_eq!(get_num_hits(query_from_text("ip:{127.0.0.11 TO *}")), 1000);
|
||||
assert_eq!(get_num_hits(query_from_text("ip:{127.0.0.10 TO *}")), 1000);
|
||||
assert_eq!(get_num_hits(query_from_text("ip:{127.0.0.21 TO *}")), 0);
|
||||
assert_eq!(get_num_hits(query_from_text("ip:{127.0.0.20 TO *}")), 0);
|
||||
assert_eq!(get_num_hits(query_from_text("ip:{127.0.0.19 TO *}")), 1000);
|
||||
assert_eq!(get_num_hits(query_from_text("ip:{* TO 127.0.0.9}")), 0);
|
||||
assert_eq!(get_num_hits(query_from_text("ip:{* TO 127.0.0.10}")), 0);
|
||||
assert_eq!(get_num_hits(query_from_text("ip:{* TO 127.0.0.11}")), 1000);
|
||||
|
||||
// Inclusive/Exclusive range
|
||||
assert_eq!(
|
||||
get_num_hits(query_from_text("ip:[127.0.0.1 TO 127.0.0.20}")),
|
||||
1000
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
get_num_hits(query_from_text("ip:{127.0.0.1 TO 127.0.0.20]")),
|
||||
2000
|
||||
);
|
||||
|
||||
// Intersection
|
||||
assert_eq!(
|
||||
get_num_hits(query_from_text(
|
||||
"text:BLUBBER AND ip:[127.0.0.10 TO 127.0.0.10]"
|
||||
)),
|
||||
1000
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
get_num_hits(query_from_text(
|
||||
"text:BLOBBER AND ip:[127.0.0.10 TO 127.0.0.10]"
|
||||
)),
|
||||
0
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
get_num_hits(query_from_text(
|
||||
"text:BLOBBER AND ip:[127.0.0.20 TO 127.0.0.20]"
|
||||
)),
|
||||
1000
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
get_num_hits(query_from_text(
|
||||
"text:BLUBBER AND ip:[127.0.0.20 TO 127.0.0.20]"
|
||||
)),
|
||||
0
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,595 +0,0 @@
|
||||
//! IP Fastfields support efficient scanning for range queries.
|
||||
//! We use this variant only if the fastfield exists, otherwise the default in `range_query` is
|
||||
//! used, which uses the term dictionary + postings.
|
||||
|
||||
use std::net::Ipv6Addr;
|
||||
use std::ops::{Bound, RangeInclusive};
|
||||
use std::sync::Arc;
|
||||
|
||||
use common::BinarySerializable;
|
||||
use fastfield_codecs::{Column, MonotonicallyMappableToU128};
|
||||
|
||||
use super::range_query::map_bound;
|
||||
use super::{ConstScorer, Explanation, Scorer, Weight};
|
||||
use crate::schema::{Cardinality, Field};
|
||||
use crate::{DocId, DocSet, Score, SegmentReader, TantivyError, TERMINATED};
|
||||
|
||||
/// `IPFastFieldRangeWeight` uses the ip address fast field to execute range queries.
|
||||
pub struct IPFastFieldRangeWeight {
|
||||
field: Field,
|
||||
left_bound: Bound<Ipv6Addr>,
|
||||
right_bound: Bound<Ipv6Addr>,
|
||||
}
|
||||
|
||||
impl IPFastFieldRangeWeight {
|
||||
pub fn new(field: Field, left_bound: &Bound<Vec<u8>>, right_bound: &Bound<Vec<u8>>) -> Self {
|
||||
let ip_from_bound_raw_data = |data: &Vec<u8>| {
|
||||
let left_ip_u128: u128 =
|
||||
u128::from_be(BinarySerializable::deserialize(&mut &data[..]).unwrap());
|
||||
Ipv6Addr::from_u128(left_ip_u128)
|
||||
};
|
||||
let left_bound = map_bound(left_bound, &ip_from_bound_raw_data);
|
||||
let right_bound = map_bound(right_bound, &ip_from_bound_raw_data);
|
||||
Self {
|
||||
field,
|
||||
left_bound,
|
||||
right_bound,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Weight for IPFastFieldRangeWeight {
|
||||
fn scorer(&self, reader: &SegmentReader, boost: Score) -> crate::Result<Box<dyn Scorer>> {
|
||||
let field_type = reader.schema().get_field_entry(self.field).field_type();
|
||||
match field_type.fastfield_cardinality().unwrap() {
|
||||
Cardinality::SingleValue => {
|
||||
let ip_addr_fast_field = reader.fast_fields().ip_addr(self.field)?;
|
||||
let value_range = bound_to_value_range(
|
||||
&self.left_bound,
|
||||
&self.right_bound,
|
||||
ip_addr_fast_field.as_ref(),
|
||||
);
|
||||
let docset = IpRangeDocSet::new(value_range, ip_addr_fast_field);
|
||||
Ok(Box::new(ConstScorer::new(docset, boost)))
|
||||
}
|
||||
Cardinality::MultiValues => unimplemented!(),
|
||||
}
|
||||
}
|
||||
|
||||
fn explain(&self, reader: &SegmentReader, doc: DocId) -> crate::Result<Explanation> {
|
||||
let mut scorer = self.scorer(reader, 1.0)?;
|
||||
if scorer.seek(doc) != doc {
|
||||
return Err(TantivyError::InvalidArgument(format!(
|
||||
"Document #({}) does not match",
|
||||
doc
|
||||
)));
|
||||
}
|
||||
let explanation = Explanation::new("Const", scorer.score());
|
||||
|
||||
Ok(explanation)
|
||||
}
|
||||
}
|
||||
|
||||
fn bound_to_value_range(
|
||||
left_bound: &Bound<Ipv6Addr>,
|
||||
right_bound: &Bound<Ipv6Addr>,
|
||||
column: &dyn Column<Ipv6Addr>,
|
||||
) -> RangeInclusive<Ipv6Addr> {
|
||||
let start_value = match left_bound {
|
||||
Bound::Included(ip_addr) => *ip_addr,
|
||||
Bound::Excluded(ip_addr) => Ipv6Addr::from(ip_addr.to_u128() + 1),
|
||||
Bound::Unbounded => column.min_value(),
|
||||
};
|
||||
|
||||
let end_value = match right_bound {
|
||||
Bound::Included(ip_addr) => *ip_addr,
|
||||
Bound::Excluded(ip_addr) => Ipv6Addr::from(ip_addr.to_u128() - 1),
|
||||
Bound::Unbounded => column.max_value(),
|
||||
};
|
||||
start_value..=end_value
|
||||
}
|
||||
|
||||
/// Helper to have a cursor over a vec of docids
|
||||
struct VecCursor {
|
||||
docs: Vec<u32>,
|
||||
current_pos: usize,
|
||||
}
|
||||
impl VecCursor {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
docs: Vec::with_capacity(32),
|
||||
current_pos: 0,
|
||||
}
|
||||
}
|
||||
fn next(&mut self) -> Option<u32> {
|
||||
self.current_pos += 1;
|
||||
self.current()
|
||||
}
|
||||
#[inline]
|
||||
fn current(&self) -> Option<u32> {
|
||||
self.docs.get(self.current_pos).map(|el| *el as u32)
|
||||
}
|
||||
|
||||
fn get_cleared_data(&mut self) -> &mut Vec<u32> {
|
||||
self.docs.clear();
|
||||
self.current_pos = 0;
|
||||
&mut self.docs
|
||||
}
|
||||
|
||||
fn is_empty(&self) -> bool {
|
||||
self.current_pos >= self.docs.len()
|
||||
}
|
||||
}
|
||||
|
||||
struct IpRangeDocSet {
|
||||
/// The range filter on the values.
|
||||
value_range: RangeInclusive<Ipv6Addr>,
|
||||
ip_addr_fast_field: Arc<dyn Column<Ipv6Addr>>,
|
||||
/// The next docid start range to fetch (inclusive).
|
||||
next_fetch_start: u32,
|
||||
/// Number of docs range checked in a batch.
|
||||
///
|
||||
/// There are two patterns.
|
||||
/// - We do a full scan. => We can load large chunks. We don't know in advance if seek call
|
||||
/// will come, so we start with small chunks
|
||||
/// - We load docs, interspersed with seek calls. When there are big jumps in the seek, we
|
||||
/// should load small chunks. When the seeks are small, we can employ the same strategy as on a
|
||||
/// full scan.
|
||||
fetch_horizon: u32,
|
||||
/// Current batch of loaded docs.
|
||||
loaded_docs: VecCursor,
|
||||
last_seek_pos_opt: Option<u32>,
|
||||
}
|
||||
|
||||
const DEFALT_FETCH_HORIZON: u32 = 128;
|
||||
impl IpRangeDocSet {
|
||||
fn new(
|
||||
value_range: RangeInclusive<Ipv6Addr>,
|
||||
ip_addr_fast_field: Arc<dyn Column<Ipv6Addr>>,
|
||||
) -> Self {
|
||||
let mut ip_range_docset = Self {
|
||||
value_range,
|
||||
ip_addr_fast_field,
|
||||
loaded_docs: VecCursor::new(),
|
||||
next_fetch_start: 0,
|
||||
fetch_horizon: DEFALT_FETCH_HORIZON,
|
||||
last_seek_pos_opt: None,
|
||||
};
|
||||
ip_range_docset.reset_fetch_range();
|
||||
ip_range_docset.fetch_block();
|
||||
ip_range_docset
|
||||
}
|
||||
|
||||
fn reset_fetch_range(&mut self) {
|
||||
self.fetch_horizon = DEFALT_FETCH_HORIZON;
|
||||
}
|
||||
|
||||
/// Returns true if more data could be fetched
|
||||
fn fetch_block(&mut self) {
|
||||
const MAX_HORIZON: u32 = 100_000;
|
||||
while self.loaded_docs.is_empty() {
|
||||
let finished_to_end = self.fetch_horizon(self.fetch_horizon);
|
||||
if finished_to_end {
|
||||
break;
|
||||
}
|
||||
// Fetch more data, increase horizon. Horizon only gets reset when doing a seek.
|
||||
self.fetch_horizon = (self.fetch_horizon * 2).min(MAX_HORIZON);
|
||||
}
|
||||
}
|
||||
|
||||
/// check if the distance between the seek calls is large
|
||||
fn is_last_seek_distance_large(&self, new_seek: DocId) -> bool {
|
||||
if let Some(last_seek_pos) = self.last_seek_pos_opt {
|
||||
(new_seek - last_seek_pos) >= 128
|
||||
} else {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetches a block for docid range [next_fetch_start .. next_fetch_start + HORIZON]
|
||||
fn fetch_horizon(&mut self, horizon: u32) -> bool {
|
||||
let mut finished_to_end = false;
|
||||
|
||||
let limit = self.ip_addr_fast_field.num_vals();
|
||||
let mut end = self.next_fetch_start + horizon;
|
||||
if end >= limit {
|
||||
end = limit;
|
||||
finished_to_end = true;
|
||||
}
|
||||
|
||||
let data = self.loaded_docs.get_cleared_data();
|
||||
self.ip_addr_fast_field.get_positions_for_value_range(
|
||||
self.value_range.clone(),
|
||||
self.next_fetch_start..end,
|
||||
data,
|
||||
);
|
||||
self.next_fetch_start = end;
|
||||
finished_to_end
|
||||
}
|
||||
}
|
||||
|
||||
impl DocSet for IpRangeDocSet {
|
||||
#[inline]
|
||||
fn advance(&mut self) -> DocId {
|
||||
if let Some(docid) = self.loaded_docs.next() {
|
||||
docid as u32
|
||||
} else {
|
||||
if self.next_fetch_start >= self.ip_addr_fast_field.num_vals() as u32 {
|
||||
return TERMINATED;
|
||||
}
|
||||
self.fetch_block();
|
||||
self.loaded_docs.current().unwrap_or(TERMINATED)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn doc(&self) -> DocId {
|
||||
self.loaded_docs
|
||||
.current()
|
||||
.map(|el| el as u32)
|
||||
.unwrap_or(TERMINATED)
|
||||
}
|
||||
|
||||
/// Advances the `DocSet` forward until reaching the target, or going to the
|
||||
/// lowest [`DocId`] greater than the target.
|
||||
///
|
||||
/// If the end of the `DocSet` is reached, [`TERMINATED`] is returned.
|
||||
///
|
||||
/// Calling `.seek(target)` on a terminated `DocSet` is legal. Implementation
|
||||
/// of `DocSet` should support it.
|
||||
///
|
||||
/// Calling `seek(TERMINATED)` is also legal and is the normal way to consume a `DocSet`.
|
||||
fn seek(&mut self, target: DocId) -> DocId {
|
||||
if self.is_last_seek_distance_large(target) {
|
||||
self.reset_fetch_range();
|
||||
}
|
||||
if target > self.next_fetch_start {
|
||||
self.next_fetch_start = target;
|
||||
}
|
||||
let mut doc = self.doc();
|
||||
debug_assert!(doc <= target);
|
||||
while doc < target {
|
||||
doc = self.advance();
|
||||
}
|
||||
self.last_seek_pos_opt = Some(target);
|
||||
doc
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> u32 {
|
||||
0 // heuristic possible by checking number of hits when fetching a block
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use proptest::prelude::ProptestConfig;
|
||||
use proptest::strategy::Strategy;
|
||||
use proptest::{prop_oneof, proptest};
|
||||
|
||||
use super::*;
|
||||
use crate::collector::Count;
|
||||
use crate::query::QueryParser;
|
||||
use crate::schema::{Schema, FAST, INDEXED, STORED, STRING};
|
||||
use crate::Index;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Doc {
|
||||
pub id: String,
|
||||
pub ip: Ipv6Addr,
|
||||
}
|
||||
|
||||
fn operation_strategy() -> impl Strategy<Value = Doc> {
|
||||
prop_oneof![
|
||||
(0u64..100u64).prop_map(doc_from_id_1),
|
||||
(1u64..100u64).prop_map(doc_from_id_2),
|
||||
]
|
||||
}
|
||||
|
||||
pub fn doc_from_id_1(id: u64) -> Doc {
|
||||
Doc {
|
||||
// ip != id
|
||||
id: id.to_string(),
|
||||
ip: Ipv6Addr::from_u128(id as u128),
|
||||
}
|
||||
}
|
||||
fn doc_from_id_2(id: u64) -> Doc {
|
||||
Doc {
|
||||
// ip != id
|
||||
id: (id - 1).to_string(),
|
||||
ip: Ipv6Addr::from_u128(id as u128),
|
||||
}
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#![proptest_config(ProptestConfig::with_cases(10))]
|
||||
#[test]
|
||||
fn test_ip_range_for_docs_prop(ops in proptest::collection::vec(operation_strategy(), 1..1000)) {
|
||||
assert!(test_ip_range_for_docs(ops).is_ok());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ip_range_regression1_test() {
|
||||
let ops = vec![
|
||||
doc_from_id_1(52),
|
||||
doc_from_id_1(63),
|
||||
doc_from_id_1(12),
|
||||
doc_from_id_2(91),
|
||||
doc_from_id_2(33),
|
||||
];
|
||||
assert!(test_ip_range_for_docs(ops).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ip_range_regression2_test() {
|
||||
let ops = vec![doc_from_id_1(0)];
|
||||
assert!(test_ip_range_for_docs(ops).is_ok());
|
||||
}
|
||||
|
||||
pub fn create_index_from_docs(docs: &[Doc]) -> Index {
|
||||
let mut schema_builder = Schema::builder();
|
||||
let ip_field = schema_builder.add_ip_addr_field("ip", INDEXED | STORED | FAST);
|
||||
let text_field = schema_builder.add_text_field("id", STRING | STORED);
|
||||
let schema = schema_builder.build();
|
||||
let index = Index::create_in_ram(schema);
|
||||
|
||||
{
|
||||
let mut index_writer = index.writer(3_000_000).unwrap();
|
||||
for doc in docs.iter() {
|
||||
index_writer
|
||||
.add_document(doc!(
|
||||
ip_field => doc.ip,
|
||||
text_field => doc.id.to_string(),
|
||||
))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
index_writer.commit().unwrap();
|
||||
}
|
||||
index
|
||||
}
|
||||
|
||||
fn test_ip_range_for_docs(docs: Vec<Doc>) -> crate::Result<()> {
|
||||
let index = create_index_from_docs(&docs);
|
||||
let reader = index.reader().unwrap();
|
||||
let searcher = reader.searcher();
|
||||
|
||||
let get_num_hits = |query| searcher.search(&query, &(Count)).unwrap();
|
||||
let query_from_text = |text: &str| {
|
||||
QueryParser::for_index(&index, vec![])
|
||||
.parse_query(text)
|
||||
.unwrap()
|
||||
};
|
||||
|
||||
let gen_query_inclusive = |from: Ipv6Addr, to: Ipv6Addr| {
|
||||
format!("ip:[{} TO {}]", &from.to_string(), &to.to_string())
|
||||
};
|
||||
|
||||
let test_sample = |sample_docs: Vec<Doc>| {
|
||||
let mut ips: Vec<Ipv6Addr> = sample_docs.iter().map(|doc| doc.ip).collect();
|
||||
ips.sort();
|
||||
let expected_num_hits = docs
|
||||
.iter()
|
||||
.filter(|doc| (ips[0]..=ips[1]).contains(&doc.ip))
|
||||
.count();
|
||||
|
||||
let query = gen_query_inclusive(ips[0], ips[1]);
|
||||
assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
|
||||
|
||||
// Intersection search
|
||||
let id_filter = sample_docs[0].id.to_string();
|
||||
let expected_num_hits = docs
|
||||
.iter()
|
||||
.filter(|doc| (ips[0]..=ips[1]).contains(&doc.ip) && doc.id == id_filter)
|
||||
.count();
|
||||
let query = format!("{} AND id:{}", query, &id_filter);
|
||||
assert_eq!(get_num_hits(query_from_text(&query)), expected_num_hits);
|
||||
};
|
||||
|
||||
test_sample(vec![docs[0].clone(), docs[0].clone()]);
|
||||
if docs.len() > 1 {
|
||||
test_sample(vec![docs[0].clone(), docs[1].clone()]);
|
||||
test_sample(vec![docs[1].clone(), docs[1].clone()]);
|
||||
}
|
||||
if docs.len() > 2 {
|
||||
test_sample(vec![docs[1].clone(), docs[2].clone()]);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "unstable"))]
|
||||
mod bench {
|
||||
|
||||
use rand::{thread_rng, Rng};
|
||||
use test::Bencher;
|
||||
|
||||
use super::tests::*;
|
||||
use super::*;
|
||||
use crate::collector::Count;
|
||||
use crate::query::QueryParser;
|
||||
use crate::Index;
|
||||
|
||||
fn get_index_0_to_100() -> Index {
|
||||
let mut rng = thread_rng();
|
||||
let num_vals = 100_000;
|
||||
let docs: Vec<_> = (0..num_vals)
|
||||
.map(|_i| {
|
||||
let id = if rng.gen_bool(0.01) {
|
||||
"veryfew".to_string() // 1%
|
||||
} else if rng.gen_bool(0.1) {
|
||||
"few".to_string() // 9%
|
||||
} else {
|
||||
"many".to_string() // 90%
|
||||
};
|
||||
Doc {
|
||||
id: id,
|
||||
// Multiply by 1000, so that we create many buckets in the compact space
|
||||
ip: Ipv6Addr::from_u128(rng.gen_range(0..100) * 1000),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let index = create_index_from_docs(&docs);
|
||||
index
|
||||
}
|
||||
fn excute_query(
|
||||
start_inclusive: Ipv6Addr,
|
||||
end_inclusive: Ipv6Addr,
|
||||
suffix: &str,
|
||||
index: &Index,
|
||||
) -> usize {
|
||||
let gen_query_inclusive = |from: Ipv6Addr, to: Ipv6Addr| {
|
||||
format!(
|
||||
"ip:[{} TO {}] {}",
|
||||
&from.to_string(),
|
||||
&to.to_string(),
|
||||
suffix
|
||||
)
|
||||
};
|
||||
|
||||
let query = gen_query_inclusive(start_inclusive, end_inclusive);
|
||||
let query_from_text = |text: &str| {
|
||||
QueryParser::for_index(&index, vec![])
|
||||
.parse_query(text)
|
||||
.unwrap()
|
||||
};
|
||||
let query = query_from_text(&query);
|
||||
let reader = index.reader().unwrap();
|
||||
let searcher = reader.searcher();
|
||||
searcher.search(&query, &(Count)).unwrap()
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_90_percent(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(90 * 1000);
|
||||
|
||||
excute_query(start, end, "", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_10_percent(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query(start, end, "", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_1_percent(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(10 * 1000);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query(start, end, "", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_10_percent_intersect_with_10_percent(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query(start, end, "AND id:few", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_1_percent_intersect_with_10_percent(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(10 * 1000);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query(start, end, "AND id:few", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_1_percent_intersect_with_90_percent(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(10 * 1000);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query(start, end, "AND id:many", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_1_percent_intersect_with_1_percent(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(10 * 1000);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query(start, end, "AND id:veryfew", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_10_percent_intersect_with_90_percent(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(10 * 1000);
|
||||
|
||||
excute_query(start, end, "AND id:many", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_90_percent_intersect_with_90_percent(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(90 * 1000);
|
||||
|
||||
excute_query(start, end, "AND id:many", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_90_percent_intersect_with_10_percent(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(90 * 1000);
|
||||
|
||||
excute_query(start, end, "AND id:few", &index)
|
||||
});
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_ip_range_hit_90_percent_intersect_with_1_percent(bench: &mut Bencher) {
|
||||
let index = get_index_0_to_100();
|
||||
|
||||
bench.iter(|| {
|
||||
let start = Ipv6Addr::from_u128(0);
|
||||
let end = Ipv6Addr::from_u128(90 * 1000);
|
||||
|
||||
excute_query(start, end, "AND id:veryfew", &index)
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -115,7 +115,7 @@ mod tests {
|
||||
pub fn test_term_set_query() -> crate::Result<()> {
|
||||
let mut schema_builder = Schema::builder();
|
||||
let field1 = schema_builder.add_text_field("field1", TEXT);
|
||||
let field2 = schema_builder.add_text_field("field2", TEXT);
|
||||
let field2 = schema_builder.add_text_field("field1", TEXT);
|
||||
let schema = schema_builder.build();
|
||||
let index = Index::create_in_ram(schema);
|
||||
{
|
||||
|
||||
@@ -124,70 +124,3 @@ impl Query for TermQuery {
|
||||
visitor(&self.term, false);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::net::{IpAddr, Ipv6Addr};
|
||||
use std::str::FromStr;
|
||||
|
||||
use fastfield_codecs::MonotonicallyMappableToU128;
|
||||
|
||||
use crate::collector::{Count, TopDocs};
|
||||
use crate::query::{Query, QueryParser, TermQuery};
|
||||
use crate::schema::{IndexRecordOption, IntoIpv6Addr, Schema, INDEXED, STORED};
|
||||
use crate::{doc, Index, Term};
|
||||
|
||||
#[test]
|
||||
fn search_ip_test() {
|
||||
let mut schema_builder = Schema::builder();
|
||||
let ip_field = schema_builder.add_ip_addr_field("ip", INDEXED | STORED);
|
||||
let schema = schema_builder.build();
|
||||
let index = Index::create_in_ram(schema);
|
||||
let ip_addr_1 = IpAddr::from_str("127.0.0.1").unwrap().into_ipv6_addr();
|
||||
let ip_addr_2 = Ipv6Addr::from_u128(10);
|
||||
|
||||
{
|
||||
let mut index_writer = index.writer(3_000_000).unwrap();
|
||||
index_writer
|
||||
.add_document(doc!(
|
||||
ip_field => ip_addr_1
|
||||
))
|
||||
.unwrap();
|
||||
index_writer
|
||||
.add_document(doc!(
|
||||
ip_field => ip_addr_2
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
index_writer.commit().unwrap();
|
||||
}
|
||||
let reader = index.reader().unwrap();
|
||||
let searcher = reader.searcher();
|
||||
|
||||
let assert_single_hit = |query| {
|
||||
let (_top_docs, count) = searcher
|
||||
.search(&query, &(TopDocs::with_limit(2), Count))
|
||||
.unwrap();
|
||||
assert_eq!(count, 1);
|
||||
};
|
||||
let query_from_text = |text: String| {
|
||||
QueryParser::for_index(&index, vec![ip_field])
|
||||
.parse_query(&text)
|
||||
.unwrap()
|
||||
};
|
||||
|
||||
let query_from_ip = |ip_addr| -> Box<dyn Query> {
|
||||
Box::new(TermQuery::new(
|
||||
Term::from_field_ip_addr(ip_field, ip_addr),
|
||||
IndexRecordOption::Basic,
|
||||
))
|
||||
};
|
||||
|
||||
assert_single_hit(query_from_ip(ip_addr_1));
|
||||
assert_single_hit(query_from_ip(ip_addr_2));
|
||||
assert_single_hit(query_from_text("127.0.0.1".to_string()));
|
||||
assert_single_hit(query_from_text("\"127.0.0.1\"".to_string()));
|
||||
assert_single_hit(query_from_text(format!("\"{}\"", ip_addr_1)));
|
||||
assert_single_hit(query_from_text(format!("\"{}\"", ip_addr_2)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -263,7 +263,8 @@ impl InnerIndexReader {
|
||||
/// It controls when a new version of the index should be loaded and lends
|
||||
/// you instances of `Searcher` for the last loaded version.
|
||||
///
|
||||
/// `IndexReader` just wraps an `Arc`.
|
||||
/// `Clone` does not clone the different pool of searcher. `IndexReader`
|
||||
/// just wraps an `Arc`.
|
||||
#[derive(Clone)]
|
||||
pub struct IndexReader {
|
||||
inner: Arc<InnerIndexReader>,
|
||||
@@ -293,6 +294,9 @@ impl IndexReader {
|
||||
///
|
||||
/// This method should be called every single time a search
|
||||
/// query is performed.
|
||||
/// The searchers are taken from a pool of `num_searchers` searchers.
|
||||
/// If no searcher is available
|
||||
/// this may block.
|
||||
///
|
||||
/// The same searcher must be used for a given query, as it ensures
|
||||
/// the use of a consistent segment set.
|
||||
|
||||
@@ -3,7 +3,7 @@ use std::ops::BitOr;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::flags::{FastFlag, IndexedFlag, SchemaFlagList, StoredFlag};
|
||||
/// Define how a bytes field should be handled by tantivy.
|
||||
/// Define how an a bytes field should be handled by tantivy.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(from = "BytesOptionsDeser")]
|
||||
pub struct BytesOptions {
|
||||
|
||||
@@ -1,105 +1,34 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::io::{self, Read, Write};
|
||||
use std::net::Ipv6Addr;
|
||||
use std::sync::Arc;
|
||||
use std::{fmt, mem};
|
||||
use std::mem;
|
||||
|
||||
use common::{BinarySerializable, VInt};
|
||||
use itertools::Either;
|
||||
use yoke::erased::ErasedArcCart;
|
||||
use yoke::Yoke;
|
||||
|
||||
use super::*;
|
||||
use crate::schema::value::MaybeOwnedString;
|
||||
use crate::tokenizer::PreTokenizedString;
|
||||
use crate::DateTime;
|
||||
|
||||
/// A group of FieldValue sharing an underlying storage
|
||||
///
|
||||
/// Or a single owned FieldValue.
|
||||
#[derive(Clone)]
|
||||
enum FieldValueGroup {
|
||||
Single(FieldValue<'static>),
|
||||
Group(Yoke<VecFieldValue<'static>, ErasedArcCart>),
|
||||
}
|
||||
|
||||
// this NewType is required to make it possible to yoke a vec with non 'static inner values.
|
||||
#[derive(yoke::Yokeable, Clone)]
|
||||
struct VecFieldValue<'a>(Vec<FieldValue<'a>>);
|
||||
|
||||
impl<'a> std::ops::Deref for VecFieldValue<'a> {
|
||||
type Target = Vec<FieldValue<'a>>;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> From<Vec<FieldValue<'a>>> for VecFieldValue<'a> {
|
||||
fn from(field_values: Vec<FieldValue>) -> VecFieldValue {
|
||||
VecFieldValue(field_values)
|
||||
}
|
||||
}
|
||||
|
||||
impl FieldValueGroup {
|
||||
fn iter(&self) -> impl Iterator<Item = &FieldValue> {
|
||||
match self {
|
||||
FieldValueGroup::Single(field_value) => Either::Left(std::iter::once(field_value)),
|
||||
FieldValueGroup::Group(field_values) => Either::Right(field_values.get().iter()),
|
||||
}
|
||||
}
|
||||
|
||||
fn count(&self) -> usize {
|
||||
match self {
|
||||
FieldValueGroup::Single(_) => 1,
|
||||
FieldValueGroup::Group(field_values) => field_values.get().len(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Vec<FieldValue<'static>>> for FieldValueGroup {
|
||||
fn from(field_values: Vec<FieldValue<'static>>) -> FieldValueGroup {
|
||||
FieldValueGroup::Group(
|
||||
Yoke::new_always_owned(field_values.into())
|
||||
.wrap_cart_in_arc()
|
||||
.erase_arc_cart(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Tantivy's Document is the object that can
|
||||
/// be indexed and then searched for.
|
||||
///
|
||||
/// Documents are fundamentally a collection of unordered couples `(field, value)`.
|
||||
/// In this list, one field may appear more than once.
|
||||
#[derive(Clone, Default)]
|
||||
// TODO bring back Ser/De and Debug
|
||||
//#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, Default)]
|
||||
//#[serde(bound(deserialize = "'static: 'de, 'de: 'static"))]
|
||||
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, Default)]
|
||||
pub struct Document {
|
||||
field_values: Vec<FieldValueGroup>,
|
||||
field_values: Vec<FieldValue>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for Document {
|
||||
fn fmt(&self, _: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
todo!()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Vec<FieldValue<'static>>> for Document {
|
||||
fn from(field_values: Vec<FieldValue<'static>>) -> Self {
|
||||
let field_values = vec![field_values.into()];
|
||||
impl From<Vec<FieldValue>> for Document {
|
||||
fn from(field_values: Vec<FieldValue>) -> Self {
|
||||
Document { field_values }
|
||||
}
|
||||
}
|
||||
impl PartialEq for Document {
|
||||
fn eq(&self, other: &Document) -> bool {
|
||||
// super slow, but only here for tests
|
||||
let convert_to_comparable_map = |field_values| {
|
||||
let convert_to_comparable_map = |field_values: &[FieldValue]| {
|
||||
let mut field_value_set: HashMap<Field, HashSet<String>> = Default::default();
|
||||
for field_value in field_values {
|
||||
// for some reason rustc fails to guess the type
|
||||
let field_value: &FieldValue = field_value;
|
||||
for field_value in field_values.iter() {
|
||||
let json_val = serde_json::to_string(field_value.value()).unwrap();
|
||||
field_value_set
|
||||
.entry(field_value.field())
|
||||
@@ -109,9 +38,9 @@ impl PartialEq for Document {
|
||||
field_value_set
|
||||
};
|
||||
let self_field_values: HashMap<Field, HashSet<String>> =
|
||||
convert_to_comparable_map(self.field_values());
|
||||
convert_to_comparable_map(&self.field_values);
|
||||
let other_field_values: HashMap<Field, HashSet<String>> =
|
||||
convert_to_comparable_map(other.field_values());
|
||||
convert_to_comparable_map(&other.field_values);
|
||||
self_field_values.eq(&other_field_values)
|
||||
}
|
||||
}
|
||||
@@ -119,13 +48,12 @@ impl PartialEq for Document {
|
||||
impl Eq for Document {}
|
||||
|
||||
impl IntoIterator for Document {
|
||||
type Item = FieldValue<'static>;
|
||||
type Item = FieldValue;
|
||||
|
||||
type IntoIter = std::vec::IntoIter<FieldValue<'static>>;
|
||||
type IntoIter = std::vec::IntoIter<FieldValue>;
|
||||
|
||||
fn into_iter(self) -> Self::IntoIter {
|
||||
todo!()
|
||||
// self.field_values.into_iter()
|
||||
self.field_values.into_iter()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -155,7 +83,7 @@ impl Document {
|
||||
|
||||
/// Add a text field.
|
||||
pub fn add_text<S: ToString>(&mut self, field: Field, text: S) {
|
||||
let value = Value::Str(MaybeOwnedString::from_string(text.to_string()));
|
||||
let value = Value::Str(text.to_string());
|
||||
self.add_field_value(field, value);
|
||||
}
|
||||
|
||||
@@ -169,11 +97,6 @@ impl Document {
|
||||
self.add_field_value(field, value);
|
||||
}
|
||||
|
||||
/// Add a IP address field. Internally only Ipv6Addr is used.
|
||||
pub fn add_ip_addr(&mut self, field: Field, value: Ipv6Addr) {
|
||||
self.add_field_value(field, value);
|
||||
}
|
||||
|
||||
/// Add a i64 field
|
||||
pub fn add_i64(&mut self, field: Field, value: i64) {
|
||||
self.add_field_value(field, value);
|
||||
@@ -209,35 +132,15 @@ impl Document {
|
||||
}
|
||||
|
||||
/// Add a (field, value) to the document.
|
||||
pub fn add_field_value<T: Into<Value<'static>>>(&mut self, field: Field, typed_val: T) {
|
||||
pub fn add_field_value<T: Into<Value>>(&mut self, field: Field, typed_val: T) {
|
||||
let value = typed_val.into();
|
||||
let field_value = FieldValue { field, value };
|
||||
self.field_values.push(FieldValueGroup::Single(field_value));
|
||||
}
|
||||
|
||||
/// Add multiple borrowed values, also taking the container they're borrowing from
|
||||
// TODO add a try_ variant?
|
||||
pub fn add_borrowed_values<T, F>(&mut self, storage: T, f: F)
|
||||
where
|
||||
T: Send + Sync + 'static,
|
||||
F: FnOnce(&T) -> Vec<FieldValue>,
|
||||
{
|
||||
let yoke =
|
||||
Yoke::attach_to_cart(Arc::new(storage), |storage| f(storage).into()).erase_arc_cart();
|
||||
|
||||
self.field_values.push(FieldValueGroup::Group(yoke));
|
||||
self.field_values.push(field_value);
|
||||
}
|
||||
|
||||
/// field_values accessor
|
||||
pub fn field_values(&self) -> impl Iterator<Item = &FieldValue> {
|
||||
self.field_values.iter().flat_map(|group| group.iter())
|
||||
}
|
||||
|
||||
/// Return the total number of values
|
||||
///
|
||||
/// More efficient than calling `self.field_values().count()`
|
||||
pub fn value_count(&self) -> usize {
|
||||
self.field_values.iter().map(|group| group.count()).sum()
|
||||
pub fn field_values(&self) -> &[FieldValue] {
|
||||
&self.field_values
|
||||
}
|
||||
|
||||
/// Sort and groups the field_values by field.
|
||||
@@ -245,7 +148,7 @@ impl Document {
|
||||
/// The result of this method is not cached and is
|
||||
/// computed on the fly when this method is called.
|
||||
pub fn get_sorted_field_values(&self) -> Vec<(Field, Vec<&Value>)> {
|
||||
let mut field_values: Vec<&FieldValue> = self.field_values().collect();
|
||||
let mut field_values: Vec<&FieldValue> = self.field_values().iter().collect();
|
||||
field_values.sort_by_key(|field_value| field_value.field());
|
||||
|
||||
let mut field_values_it = field_values.into_iter();
|
||||
@@ -280,7 +183,6 @@ impl Document {
|
||||
pub fn get_all(&self, field: Field) -> impl Iterator<Item = &Value> {
|
||||
self.field_values
|
||||
.iter()
|
||||
.flat_map(|group| group.iter())
|
||||
.filter(move |field_value| field_value.field() == field)
|
||||
.map(FieldValue::value)
|
||||
}
|
||||
@@ -289,41 +191,12 @@ impl Document {
|
||||
pub fn get_first(&self, field: Field) -> Option<&Value> {
|
||||
self.get_all(field).next()
|
||||
}
|
||||
|
||||
/// Serializes stored field values.
|
||||
pub fn serialize_stored<W: Write>(&self, schema: &Schema, writer: &mut W) -> io::Result<()> {
|
||||
let stored_field_values = || {
|
||||
self.field_values()
|
||||
.filter(|field_value| schema.get_field_entry(field_value.field()).is_stored())
|
||||
};
|
||||
let num_field_values = stored_field_values().count();
|
||||
|
||||
VInt(num_field_values as u64).serialize(writer)?;
|
||||
for field_value in stored_field_values() {
|
||||
match field_value {
|
||||
FieldValue {
|
||||
field,
|
||||
value: Value::PreTokStr(pre_tokenized_text),
|
||||
} => {
|
||||
let field_value = FieldValue {
|
||||
field: *field,
|
||||
value: Value::Str(MaybeOwnedString::from_string(
|
||||
pre_tokenized_text.text.to_string(),
|
||||
)),
|
||||
};
|
||||
field_value.serialize(writer)?;
|
||||
}
|
||||
field_value => field_value.serialize(writer)?,
|
||||
};
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl BinarySerializable for Document {
|
||||
fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
|
||||
let field_values = self.field_values();
|
||||
VInt(self.value_count() as u64).serialize(writer)?;
|
||||
VInt(field_values.len() as u64).serialize(writer)?;
|
||||
for field_value in field_values {
|
||||
field_value.serialize(writer)?;
|
||||
}
|
||||
@@ -352,7 +225,7 @@ mod tests {
|
||||
let text_field = schema_builder.add_text_field("title", TEXT);
|
||||
let mut doc = Document::default();
|
||||
doc.add_text(text_field, "My title");
|
||||
assert_eq!(doc.value_count(), 1);
|
||||
assert_eq!(doc.field_values().len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -366,7 +239,7 @@ mod tests {
|
||||
.clone(),
|
||||
);
|
||||
doc.add_text(Field::from_field_id(1), "hello");
|
||||
assert_eq!(doc.value_count(), 2);
|
||||
assert_eq!(doc.field_values().len(), 2);
|
||||
let mut payload: Vec<u8> = Vec::new();
|
||||
doc.serialize(&mut payload).unwrap();
|
||||
assert_eq!(payload.len(), 26);
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::ip_options::IpAddrOptions;
|
||||
use crate::schema::bytes_options::BytesOptions;
|
||||
use crate::schema::{
|
||||
is_valid_field_name, DateOptions, FacetOptions, FieldType, JsonObjectOptions, NumericOptions,
|
||||
@@ -61,11 +60,6 @@ impl FieldEntry {
|
||||
Self::new(field_name, FieldType::Date(date_options))
|
||||
}
|
||||
|
||||
/// Creates a new ip address field entry.
|
||||
pub fn new_ip_addr(field_name: String, ip_options: IpAddrOptions) -> FieldEntry {
|
||||
Self::new(field_name, FieldType::IpAddr(ip_options))
|
||||
}
|
||||
|
||||
/// Creates a field entry for a facet.
|
||||
pub fn new_facet(field_name: String, facet_options: FacetOptions) -> FieldEntry {
|
||||
Self::new(field_name, FieldType::Facet(facet_options))
|
||||
@@ -120,7 +114,6 @@ impl FieldEntry {
|
||||
FieldType::Facet(ref options) => options.is_stored(),
|
||||
FieldType::Bytes(ref options) => options.is_stored(),
|
||||
FieldType::JsonObject(ref options) => options.is_stored(),
|
||||
FieldType::IpAddr(ref options) => options.is_stored(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,15 +1,10 @@
|
||||
use std::net::IpAddr;
|
||||
use std::str::FromStr;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value as JsonValue;
|
||||
use thiserror::Error;
|
||||
|
||||
use super::ip_options::IpAddrOptions;
|
||||
use super::{Cardinality, IntoIpv6Addr};
|
||||
use super::Cardinality;
|
||||
use crate::schema::bytes_options::BytesOptions;
|
||||
use crate::schema::facet_options::FacetOptions;
|
||||
use crate::schema::value::MaybeOwnedString;
|
||||
use crate::schema::{
|
||||
DateOptions, Facet, IndexRecordOption, JsonObjectOptions, NumericOptions, TextFieldIndexing,
|
||||
TextOptions, Value,
|
||||
@@ -67,11 +62,9 @@ pub enum Type {
|
||||
Bytes = b'b',
|
||||
/// Leaf in a Json object.
|
||||
Json = b'j',
|
||||
/// IpAddr
|
||||
IpAddr = b'p',
|
||||
}
|
||||
|
||||
const ALL_TYPES: [Type; 10] = [
|
||||
const ALL_TYPES: [Type; 9] = [
|
||||
Type::Str,
|
||||
Type::U64,
|
||||
Type::I64,
|
||||
@@ -81,7 +74,6 @@ const ALL_TYPES: [Type; 10] = [
|
||||
Type::Facet,
|
||||
Type::Bytes,
|
||||
Type::Json,
|
||||
Type::IpAddr,
|
||||
];
|
||||
|
||||
impl Type {
|
||||
@@ -108,7 +100,6 @@ impl Type {
|
||||
Type::Facet => "Facet",
|
||||
Type::Bytes => "Bytes",
|
||||
Type::Json => "Json",
|
||||
Type::IpAddr => "IpAddr",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,7 +116,6 @@ impl Type {
|
||||
b'h' => Some(Type::Facet),
|
||||
b'b' => Some(Type::Bytes),
|
||||
b'j' => Some(Type::Json),
|
||||
b'p' => Some(Type::IpAddr),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -156,8 +146,6 @@ pub enum FieldType {
|
||||
Bytes(BytesOptions),
|
||||
/// Json object
|
||||
JsonObject(JsonObjectOptions),
|
||||
/// IpAddr field
|
||||
IpAddr(IpAddrOptions),
|
||||
}
|
||||
|
||||
impl FieldType {
|
||||
@@ -173,15 +161,9 @@ impl FieldType {
|
||||
FieldType::Facet(_) => Type::Facet,
|
||||
FieldType::Bytes(_) => Type::Bytes,
|
||||
FieldType::JsonObject(_) => Type::Json,
|
||||
FieldType::IpAddr(_) => Type::IpAddr,
|
||||
}
|
||||
}
|
||||
|
||||
/// returns true if this is an ip address field
|
||||
pub fn is_ip_addr(&self) -> bool {
|
||||
matches!(self, FieldType::IpAddr(_))
|
||||
}
|
||||
|
||||
/// returns true if the field is indexed.
|
||||
pub fn is_indexed(&self) -> bool {
|
||||
match *self {
|
||||
@@ -194,7 +176,6 @@ impl FieldType {
|
||||
FieldType::Facet(ref _facet_options) => true,
|
||||
FieldType::Bytes(ref bytes_options) => bytes_options.is_indexed(),
|
||||
FieldType::JsonObject(ref json_object_options) => json_object_options.is_indexed(),
|
||||
FieldType::IpAddr(ref ip_addr_options) => ip_addr_options.is_indexed(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -229,7 +210,6 @@ impl FieldType {
|
||||
| FieldType::F64(ref int_options)
|
||||
| FieldType::Bool(ref int_options) => int_options.is_fast(),
|
||||
FieldType::Date(ref date_options) => date_options.is_fast(),
|
||||
FieldType::IpAddr(ref ip_addr_options) => ip_addr_options.is_fast(),
|
||||
FieldType::Facet(_) => true,
|
||||
FieldType::JsonObject(_) => false,
|
||||
}
|
||||
@@ -238,11 +218,11 @@ impl FieldType {
|
||||
/// returns true if the field is fast.
|
||||
pub fn fastfield_cardinality(&self) -> Option<Cardinality> {
|
||||
match *self {
|
||||
FieldType::Bytes(ref bytes_options) => {
|
||||
bytes_options.is_fast().then_some(Cardinality::SingleValue)
|
||||
FieldType::Bytes(ref bytes_options) if bytes_options.is_fast() => {
|
||||
Some(Cardinality::SingleValue)
|
||||
}
|
||||
FieldType::Str(ref text_options) => {
|
||||
text_options.is_fast().then_some(Cardinality::MultiValues)
|
||||
FieldType::Str(ref text_options) if text_options.is_fast() => {
|
||||
Some(Cardinality::MultiValues)
|
||||
}
|
||||
FieldType::U64(ref int_options)
|
||||
| FieldType::I64(ref int_options)
|
||||
@@ -251,7 +231,7 @@ impl FieldType {
|
||||
FieldType::Date(ref date_options) => date_options.get_fastfield_cardinality(),
|
||||
FieldType::Facet(_) => Some(Cardinality::MultiValues),
|
||||
FieldType::JsonObject(_) => None,
|
||||
FieldType::IpAddr(ref ip_addr_options) => ip_addr_options.get_fastfield_cardinality(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -270,7 +250,6 @@ impl FieldType {
|
||||
FieldType::Facet(_) => false,
|
||||
FieldType::Bytes(ref bytes_options) => bytes_options.fieldnorms(),
|
||||
FieldType::JsonObject(ref _json_object_options) => false,
|
||||
FieldType::IpAddr(ref ip_addr_options) => ip_addr_options.fieldnorms(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -315,13 +294,6 @@ impl FieldType {
|
||||
FieldType::JsonObject(ref json_obj_options) => json_obj_options
|
||||
.get_text_indexing_options()
|
||||
.map(TextFieldIndexing::index_option),
|
||||
FieldType::IpAddr(ref ip_addr_options) => {
|
||||
if ip_addr_options.is_indexed() {
|
||||
Some(IndexRecordOption::Basic)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -330,7 +302,7 @@ impl FieldType {
|
||||
/// Tantivy will not try to cast values.
|
||||
/// For instance, If the json value is the integer `3` and the
|
||||
/// target field is a `Str`, this method will return an Error.
|
||||
pub fn value_from_json(&self, json: JsonValue) -> Result<Value<'static>, ValueParsingError> {
|
||||
pub fn value_from_json(&self, json: JsonValue) -> Result<Value, ValueParsingError> {
|
||||
match json {
|
||||
JsonValue::String(field_text) => {
|
||||
match self {
|
||||
@@ -342,7 +314,7 @@ impl FieldType {
|
||||
})?;
|
||||
Ok(DateTime::from_utc(dt_with_fixed_tz).into())
|
||||
}
|
||||
FieldType::Str(_) => Ok(Value::Str(MaybeOwnedString::from_string(field_text))),
|
||||
FieldType::Str(_) => Ok(Value::Str(field_text)),
|
||||
FieldType::U64(_) | FieldType::I64(_) | FieldType::F64(_) => {
|
||||
Err(ValueParsingError::TypeError {
|
||||
expected: "an integer",
|
||||
@@ -361,16 +333,6 @@ impl FieldType {
|
||||
expected: "a json object",
|
||||
json: JsonValue::String(field_text),
|
||||
}),
|
||||
FieldType::IpAddr(_) => {
|
||||
let ip_addr: IpAddr = IpAddr::from_str(&field_text).map_err(|err| {
|
||||
ValueParsingError::ParseError {
|
||||
error: err.to_string(),
|
||||
json: JsonValue::String(field_text),
|
||||
}
|
||||
})?;
|
||||
|
||||
Ok(Value::IpAddr(ip_addr.into_ipv6_addr()))
|
||||
}
|
||||
}
|
||||
}
|
||||
JsonValue::Number(field_val_num) => match self {
|
||||
@@ -418,10 +380,6 @@ impl FieldType {
|
||||
expected: "a json object",
|
||||
json: JsonValue::Number(field_val_num),
|
||||
}),
|
||||
FieldType::IpAddr(_) => Err(ValueParsingError::TypeError {
|
||||
expected: "a string with an ip addr",
|
||||
json: JsonValue::Number(field_val_num),
|
||||
}),
|
||||
},
|
||||
JsonValue::Object(json_map) => match self {
|
||||
FieldType::Str(_) => {
|
||||
|
||||
@@ -7,13 +7,12 @@ use crate::schema::{Field, Value};
|
||||
/// `FieldValue` holds together a `Field` and its `Value`.
|
||||
#[allow(missing_docs)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(bound(deserialize = "'a: 'de, 'de: 'a"))]
|
||||
pub struct FieldValue<'a> {
|
||||
pub struct FieldValue {
|
||||
pub field: Field,
|
||||
pub value: Value<'a>,
|
||||
pub value: Value,
|
||||
}
|
||||
|
||||
impl<'a> FieldValue<'a> {
|
||||
impl FieldValue {
|
||||
/// Constructor
|
||||
pub fn new(field: Field, value: Value) -> FieldValue {
|
||||
FieldValue { field, value }
|
||||
@@ -30,13 +29,13 @@ impl<'a> FieldValue<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> From<FieldValue<'a>> for Value<'a> {
|
||||
fn from(field_value: FieldValue<'a>) -> Self {
|
||||
impl From<FieldValue> for Value {
|
||||
fn from(field_value: FieldValue) -> Self {
|
||||
field_value.value
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> BinarySerializable for FieldValue<'a> {
|
||||
impl BinarySerializable for FieldValue {
|
||||
fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
|
||||
self.field.serialize(writer)?;
|
||||
self.value.serialize(writer)
|
||||
|
||||
@@ -37,8 +37,6 @@ pub struct FastFlag;
|
||||
///
|
||||
/// Fast fields can be random-accessed rapidly. Fields useful for scoring, filtering
|
||||
/// or collection should be mark as fast fields.
|
||||
///
|
||||
/// See [fast fields](`crate::fastfield`).
|
||||
pub const FAST: SchemaFlagList<FastFlag, ()> = SchemaFlagList {
|
||||
head: FastFlag,
|
||||
tail: (),
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
use std::net::{IpAddr, Ipv6Addr};
|
||||
use std::ops::BitOr;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -6,52 +5,25 @@ use serde::{Deserialize, Serialize};
|
||||
use super::flags::{FastFlag, IndexedFlag, SchemaFlagList, StoredFlag};
|
||||
use super::Cardinality;
|
||||
|
||||
/// Trait to convert into an Ipv6Addr.
|
||||
pub trait IntoIpv6Addr {
|
||||
/// Consumes the object and returns an Ipv6Addr.
|
||||
fn into_ipv6_addr(self) -> Ipv6Addr;
|
||||
}
|
||||
|
||||
impl IntoIpv6Addr for IpAddr {
|
||||
fn into_ipv6_addr(self) -> Ipv6Addr {
|
||||
match self {
|
||||
IpAddr::V4(addr) => addr.to_ipv6_mapped(),
|
||||
IpAddr::V6(addr) => addr,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Define how an ip field should be handled by tantivy.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
pub struct IpAddrOptions {
|
||||
pub struct IpOptions {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
fast: Option<Cardinality>,
|
||||
stored: bool,
|
||||
indexed: bool,
|
||||
fieldnorms: bool,
|
||||
}
|
||||
|
||||
impl IpAddrOptions {
|
||||
impl IpOptions {
|
||||
/// Returns true iff the value is a fast field.
|
||||
pub fn is_fast(&self) -> bool {
|
||||
self.fast.is_some()
|
||||
}
|
||||
|
||||
/// Returns `true` if the ip address should be stored in the doc store.
|
||||
/// Returns `true` if the json object should be stored.
|
||||
pub fn is_stored(&self) -> bool {
|
||||
self.stored
|
||||
}
|
||||
|
||||
/// Returns true iff the value is indexed and therefore searchable.
|
||||
pub fn is_indexed(&self) -> bool {
|
||||
self.indexed
|
||||
}
|
||||
|
||||
/// Returns true if and only if the value is normed.
|
||||
pub fn fieldnorms(&self) -> bool {
|
||||
self.fieldnorms
|
||||
}
|
||||
|
||||
/// Returns the cardinality of the fastfield.
|
||||
///
|
||||
/// If the field has not been declared as a fastfield, then
|
||||
@@ -60,16 +32,6 @@ impl IpAddrOptions {
|
||||
self.fast
|
||||
}
|
||||
|
||||
/// Set the field as normed.
|
||||
///
|
||||
/// Setting an integer as normed will generate
|
||||
/// the fieldnorm data for it.
|
||||
#[must_use]
|
||||
pub fn set_fieldnorms(mut self) -> Self {
|
||||
self.fieldnorms = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the field as stored
|
||||
#[must_use]
|
||||
pub fn set_stored(mut self) -> Self {
|
||||
@@ -77,19 +39,6 @@ impl IpAddrOptions {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the field as indexed.
|
||||
///
|
||||
/// Setting an ip address as indexed will generate
|
||||
/// a posting list for each value taken by the ip address.
|
||||
/// Ips are normalized to IpV6.
|
||||
///
|
||||
/// This is required for the field to be searchable.
|
||||
#[must_use]
|
||||
pub fn set_indexed(mut self) -> Self {
|
||||
self.indexed = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the field as a fast field.
|
||||
///
|
||||
/// Fast fields are designed for random access.
|
||||
@@ -103,60 +52,52 @@ impl IpAddrOptions {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<()> for IpAddrOptions {
|
||||
fn from(_: ()) -> IpAddrOptions {
|
||||
IpAddrOptions::default()
|
||||
impl From<()> for IpOptions {
|
||||
fn from(_: ()) -> IpOptions {
|
||||
IpOptions::default()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<FastFlag> for IpAddrOptions {
|
||||
impl From<FastFlag> for IpOptions {
|
||||
fn from(_: FastFlag) -> Self {
|
||||
IpAddrOptions {
|
||||
fieldnorms: false,
|
||||
indexed: false,
|
||||
IpOptions {
|
||||
stored: false,
|
||||
fast: Some(Cardinality::SingleValue),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<StoredFlag> for IpAddrOptions {
|
||||
impl From<StoredFlag> for IpOptions {
|
||||
fn from(_: StoredFlag) -> Self {
|
||||
IpAddrOptions {
|
||||
fieldnorms: false,
|
||||
indexed: false,
|
||||
IpOptions {
|
||||
stored: true,
|
||||
fast: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<IndexedFlag> for IpAddrOptions {
|
||||
impl From<IndexedFlag> for IpOptions {
|
||||
fn from(_: IndexedFlag) -> Self {
|
||||
IpAddrOptions {
|
||||
fieldnorms: true,
|
||||
indexed: true,
|
||||
IpOptions {
|
||||
stored: false,
|
||||
fast: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Into<IpAddrOptions>> BitOr<T> for IpAddrOptions {
|
||||
type Output = IpAddrOptions;
|
||||
impl<T: Into<IpOptions>> BitOr<T> for IpOptions {
|
||||
type Output = IpOptions;
|
||||
|
||||
fn bitor(self, other: T) -> IpAddrOptions {
|
||||
fn bitor(self, other: T) -> IpOptions {
|
||||
let other = other.into();
|
||||
IpAddrOptions {
|
||||
fieldnorms: self.fieldnorms | other.fieldnorms,
|
||||
indexed: self.indexed | other.indexed,
|
||||
IpOptions {
|
||||
stored: self.stored | other.stored,
|
||||
fast: self.fast.or(other.fast),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<Head, Tail> From<SchemaFlagList<Head, Tail>> for IpAddrOptions
|
||||
impl<Head, Tail> From<SchemaFlagList<Head, Tail>> for IpOptions
|
||||
where
|
||||
Head: Clone,
|
||||
Tail: Clone,
|
||||
|
||||
@@ -28,10 +28,10 @@
|
||||
//! use tantivy::schema::*;
|
||||
//! let mut schema_builder = Schema::builder();
|
||||
//! let title_options = TextOptions::default()
|
||||
//! .set_stored()
|
||||
//! .set_indexing_options(TextFieldIndexing::default()
|
||||
//! .set_tokenizer("default")
|
||||
//! .set_index_option(IndexRecordOption::WithFreqsAndPositions));
|
||||
//! .set_stored()
|
||||
//! .set_indexing_options(TextFieldIndexing::default()
|
||||
//! .set_tokenizer("default")
|
||||
//! .set_index_option(IndexRecordOption::WithFreqsAndPositions));
|
||||
//! schema_builder.add_text_field("title", title_options);
|
||||
//! let schema = schema_builder.build();
|
||||
//! ```
|
||||
@@ -45,7 +45,8 @@
|
||||
//! In the first phase, the ability to search for documents by the given field is determined by the
|
||||
//! [`IndexRecordOption`] of our [`TextOptions`].
|
||||
//!
|
||||
//! The effect of each possible setting is described more in detail in [`TextOptions`].
|
||||
//! The effect of each possible setting is described more in detail
|
||||
//! [`TextIndexingOptions`](enum.TextIndexingOptions.html).
|
||||
//!
|
||||
//! On the other hand setting the field as stored or not determines whether the field should be
|
||||
//! returned when [`Searcher::doc()`](crate::Searcher::doc) is called.
|
||||
@@ -59,8 +60,8 @@
|
||||
//! use tantivy::schema::*;
|
||||
//! let mut schema_builder = Schema::builder();
|
||||
//! let num_stars_options = NumericOptions::default()
|
||||
//! .set_stored()
|
||||
//! .set_indexed();
|
||||
//! .set_stored()
|
||||
//! .set_indexed();
|
||||
//! schema_builder.add_u64_field("num_stars", num_stars_options);
|
||||
//! let schema = schema_builder.build();
|
||||
//! ```
|
||||
@@ -78,8 +79,8 @@
|
||||
//! For convenience, it is possible to define your field indexing options by combining different
|
||||
//! flags using the `|` operator.
|
||||
//!
|
||||
//! For instance, a schema containing the two fields defined in the example above could be
|
||||
//! rewritten:
|
||||
//! For instance, a schema containing the two fields defined in the example above could be rewritten
|
||||
//! :
|
||||
//!
|
||||
//! ```
|
||||
//! use tantivy::schema::*;
|
||||
@@ -138,7 +139,7 @@ pub use self::field_type::{FieldType, Type};
|
||||
pub use self::field_value::FieldValue;
|
||||
pub use self::flags::{FAST, INDEXED, STORED};
|
||||
pub use self::index_record_option::IndexRecordOption;
|
||||
pub use self::ip_options::{IntoIpv6Addr, IpAddrOptions};
|
||||
pub use self::ip_options::IpOptions;
|
||||
pub use self::json_object_options::JsonObjectOptions;
|
||||
pub use self::named_field_document::NamedFieldDocument;
|
||||
pub use self::numeric_options::NumericOptions;
|
||||
|
||||
@@ -10,5 +10,4 @@ use crate::schema::Value;
|
||||
/// A `NamedFieldDocument` is a simple representation of a document
|
||||
/// as a `BTreeMap<String, Vec<Value>>`.
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
#[serde(bound(deserialize = "'static: 'de, 'de: 'static"))]
|
||||
pub struct NamedFieldDocument(pub BTreeMap<String, Vec<Value<'static>>>);
|
||||
pub struct NamedFieldDocument(pub BTreeMap<String, Vec<Value>>);
|
||||
|
||||
@@ -59,7 +59,7 @@ impl From<NumericOptionsDeser> for NumericOptions {
|
||||
}
|
||||
|
||||
impl NumericOptions {
|
||||
/// Returns true iff the value is stored in the doc store.
|
||||
/// Returns true iff the value is stored.
|
||||
pub fn is_stored(&self) -> bool {
|
||||
self.stored
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ use serde::ser::SerializeSeq;
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||
use serde_json::{self, Value as JsonValue};
|
||||
|
||||
use super::ip_options::IpAddrOptions;
|
||||
use super::*;
|
||||
use crate::schema::bytes_options::BytesOptions;
|
||||
use crate::schema::field_type::ValueParsingError;
|
||||
@@ -46,9 +45,13 @@ impl SchemaBuilder {
|
||||
/// Adds a new u64 field.
|
||||
/// Returns the associated field handle
|
||||
///
|
||||
/// # Panics
|
||||
/// # Caution
|
||||
///
|
||||
/// Panics when field already exists.
|
||||
/// Appending two fields with the same name
|
||||
/// will result in the shadowing of the first
|
||||
/// by the second one.
|
||||
/// The first field will get a field id
|
||||
/// but only the second one will be indexed
|
||||
pub fn add_u64_field<T: Into<NumericOptions>>(
|
||||
&mut self,
|
||||
field_name_str: &str,
|
||||
@@ -62,9 +65,13 @@ impl SchemaBuilder {
|
||||
/// Adds a new i64 field.
|
||||
/// Returns the associated field handle
|
||||
///
|
||||
/// # Panics
|
||||
/// # Caution
|
||||
///
|
||||
/// Panics when field already exists.
|
||||
/// Appending two fields with the same name
|
||||
/// will result in the shadowing of the first
|
||||
/// by the second one.
|
||||
/// The first field will get a field id
|
||||
/// but only the second one will be indexed
|
||||
pub fn add_i64_field<T: Into<NumericOptions>>(
|
||||
&mut self,
|
||||
field_name_str: &str,
|
||||
@@ -78,9 +85,13 @@ impl SchemaBuilder {
|
||||
/// Adds a new f64 field.
|
||||
/// Returns the associated field handle
|
||||
///
|
||||
/// # Panics
|
||||
/// # Caution
|
||||
///
|
||||
/// Panics when field already exists.
|
||||
/// Appending two fields with the same name
|
||||
/// will result in the shadowing of the first
|
||||
/// by the second one.
|
||||
/// The first field will get a field id
|
||||
/// but only the second one will be indexed
|
||||
pub fn add_f64_field<T: Into<NumericOptions>>(
|
||||
&mut self,
|
||||
field_name_str: &str,
|
||||
@@ -94,9 +105,13 @@ impl SchemaBuilder {
|
||||
/// Adds a new bool field.
|
||||
/// Returns the associated field handle
|
||||
///
|
||||
/// # Panics
|
||||
/// # Caution
|
||||
///
|
||||
/// Panics when field already exists.
|
||||
/// Appending two fields with the same name
|
||||
/// will result in the shadowing of the first
|
||||
/// by the second one.
|
||||
/// The first field will get a field id
|
||||
/// but only the second one will be indexed
|
||||
pub fn add_bool_field<T: Into<NumericOptions>>(
|
||||
&mut self,
|
||||
field_name_str: &str,
|
||||
@@ -112,9 +127,13 @@ impl SchemaBuilder {
|
||||
/// Internally, Tantivy simply stores dates as i64 UTC timestamps,
|
||||
/// while the user supplies DateTime values for convenience.
|
||||
///
|
||||
/// # Panics
|
||||
/// # Caution
|
||||
///
|
||||
/// Panics when field already exists.
|
||||
/// Appending two fields with the same name
|
||||
/// will result in the shadowing of the first
|
||||
/// by the second one.
|
||||
/// The first field will get a field id
|
||||
/// but only the second one will be indexed
|
||||
pub fn add_date_field<T: Into<DateOptions>>(
|
||||
&mut self,
|
||||
field_name_str: &str,
|
||||
@@ -125,28 +144,16 @@ impl SchemaBuilder {
|
||||
self.add_field(field_entry)
|
||||
}
|
||||
|
||||
/// Adds a ip field.
|
||||
/// Returns the associated field handle.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics when field already exists.
|
||||
pub fn add_ip_addr_field<T: Into<IpAddrOptions>>(
|
||||
&mut self,
|
||||
field_name_str: &str,
|
||||
field_options: T,
|
||||
) -> Field {
|
||||
let field_name = String::from(field_name_str);
|
||||
let field_entry = FieldEntry::new_ip_addr(field_name, field_options.into());
|
||||
self.add_field(field_entry)
|
||||
}
|
||||
|
||||
/// Adds a new text field.
|
||||
/// Returns the associated field handle
|
||||
///
|
||||
/// # Panics
|
||||
/// # Caution
|
||||
///
|
||||
/// Panics when field already exists.
|
||||
/// Appending two fields with the same name
|
||||
/// will result in the shadowing of the first
|
||||
/// by the second one.
|
||||
/// The first field will get a field id
|
||||
/// but only the second one will be indexed
|
||||
pub fn add_text_field<T: Into<TextOptions>>(
|
||||
&mut self,
|
||||
field_name_str: &str,
|
||||
@@ -200,10 +207,8 @@ impl SchemaBuilder {
|
||||
pub fn add_field(&mut self, field_entry: FieldEntry) -> Field {
|
||||
let field = Field::from_field_id(self.fields.len() as u32);
|
||||
let field_name = field_entry.name().to_string();
|
||||
if let Some(_previous_value) = self.fields_map.insert(field_name, field) {
|
||||
panic!("Field already exists in schema {}", field_entry.name());
|
||||
};
|
||||
self.fields.push(field_entry);
|
||||
self.fields_map.insert(field_name, field);
|
||||
field
|
||||
}
|
||||
|
||||
@@ -308,11 +313,7 @@ impl Schema {
|
||||
let mut field_map = BTreeMap::new();
|
||||
for (field, field_values) in doc.get_sorted_field_values() {
|
||||
let field_name = self.get_field_name(field);
|
||||
let values: Vec<Value> = field_values
|
||||
.into_iter()
|
||||
.cloned()
|
||||
.map(Value::into_owned)
|
||||
.collect();
|
||||
let values: Vec<Value> = field_values.into_iter().cloned().collect();
|
||||
field_map.insert(field_name.to_string(), values);
|
||||
}
|
||||
NamedFieldDocument(field_map)
|
||||
@@ -342,21 +343,20 @@ impl Schema {
|
||||
if let Some(field) = self.get_field(&field_name) {
|
||||
let field_entry = self.get_field_entry(field);
|
||||
let field_type = field_entry.field_type();
|
||||
// TODO rewrite this with shared allocation?
|
||||
match json_value {
|
||||
JsonValue::Array(json_items) => {
|
||||
for json_item in json_items {
|
||||
let value = field_type
|
||||
.value_from_json(json_item)
|
||||
.map_err(|e| DocParsingError::ValueError(field_name.clone(), e))?;
|
||||
doc.add_field_value(field, value.into_owned());
|
||||
doc.add_field_value(field, value);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
let value = field_type
|
||||
.value_from_json(json_value)
|
||||
.map_err(|e| DocParsingError::ValueError(field_name.clone(), e))?;
|
||||
doc.add_field_value(field, value.into_owned());
|
||||
doc.add_field_value(field, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -598,14 +598,12 @@ mod tests {
|
||||
schema_builder.add_text_field("title", TEXT);
|
||||
schema_builder.add_text_field("author", STRING);
|
||||
schema_builder.add_u64_field("count", count_options);
|
||||
schema_builder.add_ip_addr_field("ip", FAST | STORED);
|
||||
schema_builder.add_bool_field("is_read", is_read_options);
|
||||
let schema = schema_builder.build();
|
||||
let doc_json = r#"{
|
||||
"title": "my title",
|
||||
"author": "fulmicoton",
|
||||
"count": 4,
|
||||
"ip": "127.0.0.1",
|
||||
"is_read": true
|
||||
}"#;
|
||||
let doc = schema.parse_document(doc_json).unwrap();
|
||||
@@ -614,39 +612,6 @@ mod tests {
|
||||
assert_eq!(doc, doc_serdeser);
|
||||
}
|
||||
|
||||
#[test]
|
||||
pub fn test_document_to_ipv4_json() {
|
||||
let mut schema_builder = Schema::builder();
|
||||
schema_builder.add_ip_addr_field("ip", FAST | STORED);
|
||||
let schema = schema_builder.build();
|
||||
|
||||
// IpV4 loopback
|
||||
let doc_json = r#"{
|
||||
"ip": "127.0.0.1"
|
||||
}"#;
|
||||
let doc = schema.parse_document(doc_json).unwrap();
|
||||
let value: serde_json::Value = serde_json::from_str(&schema.to_json(&doc)).unwrap();
|
||||
assert_eq!(value["ip"][0], "127.0.0.1");
|
||||
|
||||
// Special case IpV6 loopback. We don't want to map that to IPv4
|
||||
let doc_json = r#"{
|
||||
"ip": "::1"
|
||||
}"#;
|
||||
let doc = schema.parse_document(doc_json).unwrap();
|
||||
|
||||
let value: serde_json::Value = serde_json::from_str(&schema.to_json(&doc)).unwrap();
|
||||
assert_eq!(value["ip"][0], "::1");
|
||||
|
||||
// testing ip address of every router in the world
|
||||
let doc_json = r#"{
|
||||
"ip": "192.168.0.1"
|
||||
}"#;
|
||||
let doc = schema.parse_document(doc_json).unwrap();
|
||||
|
||||
let value: serde_json::Value = serde_json::from_str(&schema.to_json(&doc)).unwrap();
|
||||
assert_eq!(value["ip"][0], "192.168.0.1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
pub fn test_document_from_nameddoc() {
|
||||
let mut schema_builder = Schema::builder();
|
||||
@@ -711,7 +676,7 @@ mod tests {
|
||||
let schema = schema_builder.build();
|
||||
{
|
||||
let doc = schema.parse_document("{}").unwrap();
|
||||
assert_eq!(doc.value_count(), 0);
|
||||
assert!(doc.field_values().is_empty());
|
||||
}
|
||||
{
|
||||
let doc = schema
|
||||
|
||||
@@ -1,15 +1,24 @@
|
||||
use std::convert::TryInto;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::net::Ipv6Addr;
|
||||
use std::{fmt, str};
|
||||
|
||||
use fastfield_codecs::MonotonicallyMappableToU128;
|
||||
|
||||
use super::Field;
|
||||
use crate::fastfield::FastValue;
|
||||
use crate::schema::{Facet, Type};
|
||||
use crate::{DatePrecision, DateTime};
|
||||
|
||||
/// Size (in bytes) of the buffer of a fast value (u64, i64, f64, or date) term.
|
||||
/// <field> + <type byte> + <value len>
|
||||
///
|
||||
/// - <field> is a big endian encoded u32 field id
|
||||
/// - <type_byte>'s most significant bit expresses whether the term is a json term or not
|
||||
/// The remaining 7 bits are used to encode the type of the value.
|
||||
/// If this is a JSON term, the type is the type of the leaf of the json.
|
||||
///
|
||||
/// - <value> is, if this is not the json term, a binary representation specific to the type.
|
||||
/// If it is a JSON Term, then it is prepended with the path that leads to this leaf value.
|
||||
const FAST_VALUE_TERM_LEN: usize = 4 + 1 + 8;
|
||||
|
||||
/// Separates the different segments of
|
||||
/// the json path.
|
||||
pub const JSON_PATH_SEGMENT_SEP: u8 = 1u8;
|
||||
@@ -27,78 +36,45 @@ pub const JSON_END_OF_PATH: u8 = 0u8;
|
||||
pub struct Term<B = Vec<u8>>(B)
|
||||
where B: AsRef<[u8]>;
|
||||
|
||||
/// The number of bytes used as metadata by `Term`.
|
||||
const TERM_METADATA_LENGTH: usize = 5;
|
||||
impl AsMut<Vec<u8>> for Term {
|
||||
fn as_mut(&mut self) -> &mut Vec<u8> {
|
||||
&mut self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl Term {
|
||||
pub(crate) fn with_capacity(capacity: usize) -> Term {
|
||||
let mut data = Vec::with_capacity(TERM_METADATA_LENGTH + capacity);
|
||||
data.resize(TERM_METADATA_LENGTH, 0u8);
|
||||
Term(data)
|
||||
}
|
||||
|
||||
pub(crate) fn with_type_and_field(typ: Type, field: Field) -> Term {
|
||||
let mut term = Self::with_capacity(8);
|
||||
term.set_field_and_type(field, typ);
|
||||
term
|
||||
}
|
||||
|
||||
fn with_bytes_and_field_and_payload(typ: Type, field: Field, bytes: &[u8]) -> Term {
|
||||
let mut term = Self::with_capacity(bytes.len());
|
||||
term.set_field_and_type(field, typ);
|
||||
term.0.extend_from_slice(bytes);
|
||||
term
|
||||
pub(crate) fn new() -> Term {
|
||||
Term(Vec::with_capacity(100))
|
||||
}
|
||||
|
||||
fn from_fast_value<T: FastValue>(field: Field, val: &T) -> Term {
|
||||
let mut term = Self::with_type_and_field(T::to_type(), field);
|
||||
let mut term = Term(vec![0u8; FAST_VALUE_TERM_LEN]);
|
||||
term.set_field(T::to_type(), field);
|
||||
term.set_u64(val.to_u64());
|
||||
term
|
||||
}
|
||||
|
||||
/// Panics when the term is not empty... ie: some value is set.
|
||||
/// Use `clear_with_field_and_type` in that case.
|
||||
///
|
||||
/// Sets field and the type.
|
||||
pub(crate) fn set_field_and_type(&mut self, field: Field, typ: Type) {
|
||||
assert!(self.is_empty());
|
||||
self.0[0..4].clone_from_slice(field.field_id().to_be_bytes().as_ref());
|
||||
self.0[4] = typ.to_code();
|
||||
}
|
||||
|
||||
/// Is empty if there are no value bytes.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.0.len() == TERM_METADATA_LENGTH
|
||||
}
|
||||
|
||||
/// Builds a term given a field, and a `Ipv6Addr`-value
|
||||
pub fn from_field_ip_addr(field: Field, ip_addr: Ipv6Addr) -> Term {
|
||||
let mut term = Self::with_type_and_field(Type::IpAddr, field);
|
||||
term.set_ip_addr(ip_addr);
|
||||
term
|
||||
}
|
||||
|
||||
/// Builds a term given a field, and a `u64`-value
|
||||
/// Builds a term given a field, and a u64-value
|
||||
pub fn from_field_u64(field: Field, val: u64) -> Term {
|
||||
Term::from_fast_value(field, &val)
|
||||
}
|
||||
|
||||
/// Builds a term given a field, and a `i64`-value
|
||||
/// Builds a term given a field, and a i64-value
|
||||
pub fn from_field_i64(field: Field, val: i64) -> Term {
|
||||
Term::from_fast_value(field, &val)
|
||||
}
|
||||
|
||||
/// Builds a term given a field, and a `f64`-value
|
||||
/// Builds a term given a field, and a f64-value
|
||||
pub fn from_field_f64(field: Field, val: f64) -> Term {
|
||||
Term::from_fast_value(field, &val)
|
||||
}
|
||||
|
||||
/// Builds a term given a field, and a `bool`-value
|
||||
/// Builds a term given a field, and a f64-value
|
||||
pub fn from_field_bool(field: Field, val: bool) -> Term {
|
||||
Term::from_fast_value(field, &val)
|
||||
}
|
||||
|
||||
/// Builds a term given a field, and a `DateTime` value
|
||||
/// Builds a term given a field, and a DateTime value
|
||||
pub fn from_field_date(field: Field, val: DateTime) -> Term {
|
||||
Term::from_fast_value(field, &val.truncate(DatePrecision::Seconds))
|
||||
}
|
||||
@@ -106,29 +82,31 @@ impl Term {
|
||||
/// Creates a `Term` given a facet.
|
||||
pub fn from_facet(field: Field, facet: &Facet) -> Term {
|
||||
let facet_encoded_str = facet.encoded_str();
|
||||
Term::with_bytes_and_field_and_payload(Type::Facet, field, facet_encoded_str.as_bytes())
|
||||
Term::create_bytes_term(Type::Facet, field, facet_encoded_str.as_bytes())
|
||||
}
|
||||
|
||||
/// Builds a term given a field, and a string value
|
||||
pub fn from_field_text(field: Field, text: &str) -> Term {
|
||||
Term::with_bytes_and_field_and_payload(Type::Str, field, text.as_bytes())
|
||||
Term::create_bytes_term(Type::Str, field, text.as_bytes())
|
||||
}
|
||||
|
||||
fn create_bytes_term(typ: Type, field: Field, bytes: &[u8]) -> Term {
|
||||
let mut term = Term(vec![0u8; 5 + bytes.len()]);
|
||||
term.set_field(typ, field);
|
||||
term.0.extend_from_slice(bytes);
|
||||
term
|
||||
}
|
||||
|
||||
/// Builds a term bytes.
|
||||
pub fn from_field_bytes(field: Field, bytes: &[u8]) -> Term {
|
||||
Term::with_bytes_and_field_and_payload(Type::Bytes, field, bytes)
|
||||
Term::create_bytes_term(Type::Bytes, field, bytes)
|
||||
}
|
||||
|
||||
/// Removes the value_bytes and set the field and type code.
|
||||
pub(crate) fn clear_with_field_and_type(&mut self, typ: Type, field: Field) {
|
||||
self.truncate_value_bytes(0);
|
||||
self.set_field_and_type(field, typ);
|
||||
}
|
||||
|
||||
/// Removes the value_bytes and set the type code.
|
||||
pub fn clear_with_type(&mut self, typ: Type) {
|
||||
self.truncate_value_bytes(0);
|
||||
self.0[4] = typ.to_code();
|
||||
pub(crate) fn set_field(&mut self, typ: Type, field: Field) {
|
||||
self.0.clear();
|
||||
self.0
|
||||
.extend_from_slice(field.field_id().to_be_bytes().as_ref());
|
||||
self.0.push(typ.to_code());
|
||||
}
|
||||
|
||||
/// Sets a u64 value in the term.
|
||||
@@ -139,6 +117,12 @@ impl Term {
|
||||
/// the natural order of the values.
|
||||
pub fn set_u64(&mut self, val: u64) {
|
||||
self.set_fast_value(val);
|
||||
self.set_bytes(val.to_be_bytes().as_ref());
|
||||
}
|
||||
|
||||
fn set_fast_value<T: FastValue>(&mut self, val: T) {
|
||||
self.0.resize(FAST_VALUE_TERM_LEN, 0u8);
|
||||
self.set_bytes(val.to_u64().to_be_bytes().as_ref());
|
||||
}
|
||||
|
||||
/// Sets a `i64` value in the term.
|
||||
@@ -146,7 +130,7 @@ impl Term {
|
||||
self.set_fast_value(val);
|
||||
}
|
||||
|
||||
/// Sets a `DateTime` value in the term.
|
||||
/// Sets a `i64` value in the term.
|
||||
pub fn set_date(&mut self, date: DateTime) {
|
||||
self.set_fast_value(date);
|
||||
}
|
||||
@@ -161,18 +145,9 @@ impl Term {
|
||||
self.set_fast_value(val);
|
||||
}
|
||||
|
||||
fn set_fast_value<T: FastValue>(&mut self, val: T) {
|
||||
self.set_bytes(val.to_u64().to_be_bytes().as_ref());
|
||||
}
|
||||
|
||||
/// Sets a `Ipv6Addr` value in the term.
|
||||
pub fn set_ip_addr(&mut self, val: Ipv6Addr) {
|
||||
self.set_bytes(val.to_u128().to_be_bytes().as_ref());
|
||||
}
|
||||
|
||||
/// Sets the value of a `Bytes` field.
|
||||
pub fn set_bytes(&mut self, bytes: &[u8]) {
|
||||
self.truncate_value_bytes(0);
|
||||
self.0.resize(5, 0u8);
|
||||
self.0.extend(bytes);
|
||||
}
|
||||
|
||||
@@ -181,22 +156,18 @@ impl Term {
|
||||
self.set_bytes(text.as_bytes());
|
||||
}
|
||||
|
||||
/// Truncates the value bytes of the term. Value and field type stays the same.
|
||||
pub fn truncate_value_bytes(&mut self, len: usize) {
|
||||
self.0.truncate(len + TERM_METADATA_LENGTH);
|
||||
/// Removes the value_bytes and set the type code.
|
||||
pub fn clear_with_type(&mut self, typ: Type) {
|
||||
self.truncate(5);
|
||||
self.0[4] = typ.to_code();
|
||||
}
|
||||
|
||||
/// Returns the value bytes as mutable slice
|
||||
pub fn value_bytes_mut(&mut self) -> &mut [u8] {
|
||||
&mut self.0[TERM_METADATA_LENGTH..]
|
||||
/// Truncate the term right after the field and the type code.
|
||||
pub fn truncate(&mut self, len: usize) {
|
||||
self.0.truncate(len);
|
||||
}
|
||||
|
||||
/// The length of the bytes.
|
||||
pub fn len_bytes(&self) -> usize {
|
||||
self.0.len() - TERM_METADATA_LENGTH
|
||||
}
|
||||
|
||||
/// Appends value bytes to the Term.
|
||||
/// Truncate the term right after the field and the type code.
|
||||
pub fn append_bytes(&mut self, bytes: &[u8]) {
|
||||
self.0.extend_from_slice(bytes);
|
||||
}
|
||||
@@ -322,6 +293,9 @@ where B: AsRef<[u8]>
|
||||
/// Returns `None` if the field is not of string type
|
||||
/// or if the bytes are not valid utf-8.
|
||||
pub fn as_str(&self) -> Option<&str> {
|
||||
if self.as_slice().len() < 5 {
|
||||
return None;
|
||||
}
|
||||
if self.typ() != Type::Str {
|
||||
return None;
|
||||
}
|
||||
@@ -333,6 +307,9 @@ where B: AsRef<[u8]>
|
||||
/// Returns `None` if the field is not of facet type
|
||||
/// or if the bytes are not valid utf-8.
|
||||
pub fn as_facet(&self) -> Option<Facet> {
|
||||
if self.as_slice().len() < 5 {
|
||||
return None;
|
||||
}
|
||||
if self.typ() != Type::Facet {
|
||||
return None;
|
||||
}
|
||||
@@ -344,6 +321,9 @@ where B: AsRef<[u8]>
|
||||
///
|
||||
/// Returns `None` if the field is not of bytes type.
|
||||
pub fn as_bytes(&self) -> Option<&[u8]> {
|
||||
if self.as_slice().len() < 5 {
|
||||
return None;
|
||||
}
|
||||
if self.typ() != Type::Bytes {
|
||||
return None;
|
||||
}
|
||||
@@ -357,7 +337,7 @@ where B: AsRef<[u8]>
|
||||
/// If the term is a u64, its value is encoded according
|
||||
/// to `byteorder::LittleEndian`.
|
||||
pub fn value_bytes(&self) -> &[u8] {
|
||||
&self.0.as_ref()[TERM_METADATA_LENGTH..]
|
||||
&self.0.as_ref()[5..]
|
||||
}
|
||||
|
||||
/// Returns the underlying `&[u8]`.
|
||||
@@ -435,9 +415,6 @@ fn debug_value_bytes(typ: Type, bytes: &[u8], f: &mut fmt::Formatter) -> fmt::Re
|
||||
debug_value_bytes(typ, bytes, f)?;
|
||||
}
|
||||
}
|
||||
Type::IpAddr => {
|
||||
write!(f, "")?; // TODO change once we actually have IP address terms.
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -471,18 +448,6 @@ mod tests {
|
||||
assert_eq!(term.as_str(), Some("test"))
|
||||
}
|
||||
|
||||
/// Size (in bytes) of the buffer of a fast value (u64, i64, f64, or date) term.
|
||||
/// <field> + <type byte> + <value len>
|
||||
///
|
||||
/// - <field> is a big endian encoded u32 field id
|
||||
/// - <type_byte>'s most significant bit expresses whether the term is a json term or not
|
||||
/// The remaining 7 bits are used to encode the type of the value.
|
||||
/// If this is a JSON term, the type is the type of the leaf of the json.
|
||||
///
|
||||
/// - <value> is, if this is not the json term, a binary representation specific to the type.
|
||||
/// If it is a JSON Term, then it is prepended with the path that leads to this leaf value.
|
||||
const FAST_VALUE_TERM_LEN: usize = 4 + 1 + 8;
|
||||
|
||||
#[test]
|
||||
pub fn test_term_u64() {
|
||||
let mut schema_builder = Schema::builder();
|
||||
@@ -490,7 +455,7 @@ mod tests {
|
||||
let term = Term::from_field_u64(count_field, 983u64);
|
||||
assert_eq!(term.field(), count_field);
|
||||
assert_eq!(term.typ(), Type::U64);
|
||||
assert_eq!(term.as_slice().len(), FAST_VALUE_TERM_LEN);
|
||||
assert_eq!(term.as_slice().len(), super::FAST_VALUE_TERM_LEN);
|
||||
assert_eq!(term.as_u64(), Some(983u64))
|
||||
}
|
||||
|
||||
@@ -501,7 +466,7 @@ mod tests {
|
||||
let term = Term::from_field_bool(bool_field, true);
|
||||
assert_eq!(term.field(), bool_field);
|
||||
assert_eq!(term.typ(), Type::Bool);
|
||||
assert_eq!(term.as_slice().len(), FAST_VALUE_TERM_LEN);
|
||||
assert_eq!(term.as_slice().len(), super::FAST_VALUE_TERM_LEN);
|
||||
assert_eq!(term.as_bool(), Some(true))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,9 +47,7 @@ impl TextOptions {
|
||||
/// unchanged. The "default" tokenizer will store the terms as lower case and this will be
|
||||
/// reflected in the dictionary.
|
||||
///
|
||||
/// The original text can be retrieved via
|
||||
/// [`TermDictionary::ord_to_term()`](crate::termdict::TermDictionary::ord_to_term)
|
||||
/// from the dictionary.
|
||||
/// The original text can be retrieved via `ord_to_term` from the dictionary.
|
||||
#[must_use]
|
||||
pub fn set_fast(mut self) -> TextOptions {
|
||||
self.fast = true;
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
use std::fmt;
|
||||
use std::net::Ipv6Addr;
|
||||
|
||||
pub use not_safe::MaybeOwnedString;
|
||||
use serde::de::Visitor;
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||
use serde_json::Map;
|
||||
@@ -13,9 +11,9 @@ use crate::DateTime;
|
||||
/// Value represents the value of a any field.
|
||||
/// It is an enum over all over all of the possible field type.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Value<'a> {
|
||||
pub enum Value {
|
||||
/// The str type is used for any text information.
|
||||
Str(MaybeOwnedString<'a>),
|
||||
Str(String),
|
||||
/// Pre-tokenized str type,
|
||||
PreTokStr(PreTokenizedString),
|
||||
/// Unsigned 64-bits Integer `u64`
|
||||
@@ -31,38 +29,14 @@ pub enum Value<'a> {
|
||||
/// Facet
|
||||
Facet(Facet),
|
||||
/// Arbitrarily sized byte array
|
||||
// TODO allow Cow<'a, [u8]>
|
||||
Bytes(Vec<u8>),
|
||||
/// Json object value.
|
||||
// TODO allow Cow keys and borrowed values
|
||||
JsonObject(serde_json::Map<String, serde_json::Value>),
|
||||
/// IpV6 Address. Internally there is no IpV4, it needs to be converted to `Ipv6Addr`.
|
||||
IpAddr(Ipv6Addr),
|
||||
}
|
||||
|
||||
impl<'a> Value<'a> {
|
||||
/// Convert a borrowing [`Value`] to an owning one.
|
||||
pub fn into_owned(self) -> Value<'static> {
|
||||
use Value::*;
|
||||
match self {
|
||||
Str(val) => Str(MaybeOwnedString::from_string(val.into_string())),
|
||||
PreTokStr(val) => PreTokStr(val),
|
||||
U64(val) => U64(val),
|
||||
I64(val) => I64(val),
|
||||
F64(val) => F64(val),
|
||||
Bool(val) => Bool(val),
|
||||
Date(val) => Date(val),
|
||||
Facet(val) => Facet(val),
|
||||
Bytes(val) => Bytes(val),
|
||||
JsonObject(val) => JsonObject(val),
|
||||
IpAddr(val) => IpAddr(val),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl Eq for Value {}
|
||||
|
||||
impl<'a> Eq for Value<'a> {}
|
||||
|
||||
impl<'a> Serialize for Value<'a> {
|
||||
impl Serialize for Value {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where S: Serializer {
|
||||
match *self {
|
||||
@@ -74,27 +48,19 @@ impl<'a> Serialize for Value<'a> {
|
||||
Value::Bool(b) => serializer.serialize_bool(b),
|
||||
Value::Date(ref date) => time::serde::rfc3339::serialize(&date.into_utc(), serializer),
|
||||
Value::Facet(ref facet) => facet.serialize(serializer),
|
||||
Value::Bytes(ref bytes) => serializer.serialize_str(&base64::encode(bytes)),
|
||||
Value::Bytes(ref bytes) => serializer.serialize_bytes(bytes),
|
||||
Value::JsonObject(ref obj) => obj.serialize(serializer),
|
||||
Value::IpAddr(ref obj) => {
|
||||
// Ensure IpV4 addresses get serialized as IpV4, but excluding IpV6 loopback.
|
||||
if let Some(ip_v4) = obj.to_ipv4_mapped() {
|
||||
ip_v4.serialize(serializer)
|
||||
} else {
|
||||
obj.serialize(serializer)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for Value<'de> {
|
||||
impl<'de> Deserialize<'de> for Value {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where D: Deserializer<'de> {
|
||||
struct ValueVisitor;
|
||||
|
||||
impl<'de> Visitor<'de> for ValueVisitor {
|
||||
type Value = Value<'de>;
|
||||
type Value = Value;
|
||||
|
||||
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("a string or u32")
|
||||
@@ -116,13 +82,12 @@ impl<'de> Deserialize<'de> for Value<'de> {
|
||||
Ok(Value::Bool(v))
|
||||
}
|
||||
|
||||
// TODO add visit_borrowed_str
|
||||
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E> {
|
||||
Ok(Value::Str(MaybeOwnedString::from_string(v.to_owned())))
|
||||
Ok(Value::Str(v.to_owned()))
|
||||
}
|
||||
|
||||
fn visit_string<E>(self, v: String) -> Result<Self::Value, E> {
|
||||
Ok(Value::Str(MaybeOwnedString::from_string(v)))
|
||||
Ok(Value::Str(v))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -130,7 +95,7 @@ impl<'de> Deserialize<'de> for Value<'de> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Value<'a> {
|
||||
impl Value {
|
||||
/// Returns the text value, provided the value is of the `Str` type.
|
||||
/// (Returns `None` if the value is not of the `Str` type).
|
||||
pub fn as_text(&self) -> Option<&str> {
|
||||
@@ -236,99 +201,82 @@ impl<'a> Value<'a> {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the ip addr, provided the value is of the `Ip` type.
|
||||
/// (Returns None if the value is not of the `Ip` type)
|
||||
pub fn as_ip_addr(&self) -> Option<Ipv6Addr> {
|
||||
if let Value::IpAddr(val) = self {
|
||||
Some(*val)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
impl From<String> for Value {
|
||||
fn from(s: String) -> Value {
|
||||
Value::Str(s)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<String> for Value<'static> {
|
||||
fn from(s: String) -> Value<'static> {
|
||||
Value::Str(MaybeOwnedString::from_string(s))
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Ipv6Addr> for Value<'static> {
|
||||
fn from(v: Ipv6Addr) -> Value<'static> {
|
||||
Value::IpAddr(v)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<u64> for Value<'static> {
|
||||
fn from(v: u64) -> Value<'static> {
|
||||
impl From<u64> for Value {
|
||||
fn from(v: u64) -> Value {
|
||||
Value::U64(v)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<i64> for Value<'static> {
|
||||
fn from(v: i64) -> Value<'static> {
|
||||
impl From<i64> for Value {
|
||||
fn from(v: i64) -> Value {
|
||||
Value::I64(v)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<f64> for Value<'static> {
|
||||
fn from(v: f64) -> Value<'static> {
|
||||
impl From<f64> for Value {
|
||||
fn from(v: f64) -> Value {
|
||||
Value::F64(v)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<bool> for Value<'static> {
|
||||
impl From<bool> for Value {
|
||||
fn from(b: bool) -> Self {
|
||||
Value::Bool(b)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<DateTime> for Value<'static> {
|
||||
fn from(dt: DateTime) -> Value<'static> {
|
||||
impl From<DateTime> for Value {
|
||||
fn from(dt: DateTime) -> Value {
|
||||
Value::Date(dt)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> From<&'a str> for Value<'a> {
|
||||
fn from(s: &'a str) -> Value<'a> {
|
||||
Value::Str(MaybeOwnedString::from_str(s))
|
||||
impl<'a> From<&'a str> for Value {
|
||||
fn from(s: &'a str) -> Value {
|
||||
Value::Str(s.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
// TODO change lifetime to 'a
|
||||
impl<'a> From<&'a [u8]> for Value<'static> {
|
||||
fn from(bytes: &'a [u8]) -> Value<'static> {
|
||||
impl<'a> From<&'a [u8]> for Value {
|
||||
fn from(bytes: &'a [u8]) -> Value {
|
||||
Value::Bytes(bytes.to_vec())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Facet> for Value<'static> {
|
||||
fn from(facet: Facet) -> Value<'static> {
|
||||
impl From<Facet> for Value {
|
||||
fn from(facet: Facet) -> Value {
|
||||
Value::Facet(facet)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Vec<u8>> for Value<'static> {
|
||||
fn from(bytes: Vec<u8>) -> Value<'static> {
|
||||
impl From<Vec<u8>> for Value {
|
||||
fn from(bytes: Vec<u8>) -> Value {
|
||||
Value::Bytes(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<PreTokenizedString> for Value<'static> {
|
||||
fn from(pretokenized_string: PreTokenizedString) -> Value<'static> {
|
||||
impl From<PreTokenizedString> for Value {
|
||||
fn from(pretokenized_string: PreTokenizedString) -> Value {
|
||||
Value::PreTokStr(pretokenized_string)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<serde_json::Map<String, serde_json::Value>> for Value<'static> {
|
||||
fn from(json_object: serde_json::Map<String, serde_json::Value>) -> Value<'static> {
|
||||
impl From<serde_json::Map<String, serde_json::Value>> for Value {
|
||||
fn from(json_object: serde_json::Map<String, serde_json::Value>) -> Value {
|
||||
Value::JsonObject(json_object)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<serde_json::Value> for Value<'static> {
|
||||
fn from(json_value: serde_json::Value) -> Value<'static> {
|
||||
impl From<serde_json::Value> for Value {
|
||||
fn from(json_value: serde_json::Value) -> Value {
|
||||
match json_value {
|
||||
serde_json::Value::Object(json_object) => Value::JsonObject(json_object),
|
||||
_ => {
|
||||
@@ -340,12 +288,10 @@ impl From<serde_json::Value> for Value<'static> {
|
||||
|
||||
mod binary_serialize {
|
||||
use std::io::{self, Read, Write};
|
||||
use std::net::Ipv6Addr;
|
||||
|
||||
use common::{f64_to_u64, u64_to_f64, BinarySerializable};
|
||||
use fastfield_codecs::MonotonicallyMappableToU128;
|
||||
|
||||
use super::{MaybeOwnedString, Value};
|
||||
use super::Value;
|
||||
use crate::schema::Facet;
|
||||
use crate::tokenizer::PreTokenizedString;
|
||||
use crate::DateTime;
|
||||
@@ -360,19 +306,17 @@ mod binary_serialize {
|
||||
const EXT_CODE: u8 = 7;
|
||||
const JSON_OBJ_CODE: u8 = 8;
|
||||
const BOOL_CODE: u8 = 9;
|
||||
const IP_CODE: u8 = 10;
|
||||
|
||||
// extended types
|
||||
|
||||
const TOK_STR_CODE: u8 = 0;
|
||||
|
||||
impl<'a> BinarySerializable for Value<'a> {
|
||||
impl BinarySerializable for Value {
|
||||
fn serialize<W: Write>(&self, writer: &mut W) -> io::Result<()> {
|
||||
match *self {
|
||||
Value::Str(ref text) => {
|
||||
TEXT_CODE.serialize(writer)?;
|
||||
// TODO impl trait for MaybeOwnedString
|
||||
text.as_str().to_owned().serialize(writer)
|
||||
text.serialize(writer)
|
||||
}
|
||||
Value::PreTokStr(ref tok_str) => {
|
||||
EXT_CODE.serialize(writer)?;
|
||||
@@ -422,10 +366,6 @@ mod binary_serialize {
|
||||
serde_json::to_writer(writer, &map)?;
|
||||
Ok(())
|
||||
}
|
||||
Value::IpAddr(ref ip) => {
|
||||
IP_CODE.serialize(writer)?;
|
||||
ip.to_u128().serialize(writer)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -434,7 +374,7 @@ mod binary_serialize {
|
||||
match type_code {
|
||||
TEXT_CODE => {
|
||||
let text = String::deserialize(reader)?;
|
||||
Ok(Value::Str(MaybeOwnedString::from_string(text)))
|
||||
Ok(Value::Str(text))
|
||||
}
|
||||
U64_CODE => {
|
||||
let value = u64::deserialize(reader)?;
|
||||
@@ -496,11 +436,6 @@ mod binary_serialize {
|
||||
let json_map = <serde_json::Map::<String, serde_json::Value> as serde::Deserialize>::deserialize(&mut de)?;
|
||||
Ok(Value::JsonObject(json_map))
|
||||
}
|
||||
IP_CODE => {
|
||||
let value = u128::deserialize(reader)?;
|
||||
Ok(Value::IpAddr(Ipv6Addr::from_u128(value)))
|
||||
}
|
||||
|
||||
_ => Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("No field type is associated with code {:?}", type_code),
|
||||
@@ -513,52 +448,9 @@ mod binary_serialize {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::Value;
|
||||
use crate::schema::{BytesOptions, Schema};
|
||||
use crate::time::format_description::well_known::Rfc3339;
|
||||
use crate::time::OffsetDateTime;
|
||||
use crate::{DateTime, Document};
|
||||
|
||||
#[test]
|
||||
fn test_parse_bytes_doc() {
|
||||
let mut schema_builder = Schema::builder();
|
||||
let bytes_options = BytesOptions::default();
|
||||
let bytes_field = schema_builder.add_bytes_field("my_bytes", bytes_options);
|
||||
let schema = schema_builder.build();
|
||||
let mut doc = Document::default();
|
||||
doc.add_bytes(bytes_field, "this is a test".as_bytes());
|
||||
let json_string = schema.to_json(&doc);
|
||||
assert_eq!(json_string, r#"{"my_bytes":["dGhpcyBpcyBhIHRlc3Q="]}"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_empty_bytes_doc() {
|
||||
let mut schema_builder = Schema::builder();
|
||||
let bytes_options = BytesOptions::default();
|
||||
let bytes_field = schema_builder.add_bytes_field("my_bytes", bytes_options);
|
||||
let schema = schema_builder.build();
|
||||
let mut doc = Document::default();
|
||||
doc.add_bytes(bytes_field, "".as_bytes());
|
||||
let json_string = schema.to_json(&doc);
|
||||
assert_eq!(json_string, r#"{"my_bytes":[""]}"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_many_bytes_doc() {
|
||||
let mut schema_builder = Schema::builder();
|
||||
let bytes_options = BytesOptions::default();
|
||||
let bytes_field = schema_builder.add_bytes_field("my_bytes", bytes_options);
|
||||
let schema = schema_builder.build();
|
||||
let mut doc = Document::default();
|
||||
doc.add_bytes(
|
||||
bytes_field,
|
||||
"A bigger test I guess\nspanning on multiple lines\nhoping this will work".as_bytes(),
|
||||
);
|
||||
let json_string = schema.to_json(&doc);
|
||||
assert_eq!(
|
||||
json_string,
|
||||
r#"{"my_bytes":["QSBiaWdnZXIgdGVzdCBJIGd1ZXNzCnNwYW5uaW5nIG9uIG11bHRpcGxlIGxpbmVzCmhvcGluZyB0aGlzIHdpbGwgd29yaw=="]}"#
|
||||
);
|
||||
}
|
||||
use crate::DateTime;
|
||||
|
||||
#[test]
|
||||
fn test_serialize_date() {
|
||||
@@ -576,104 +468,3 @@ mod tests {
|
||||
assert_eq!(serialized_value_json, r#""1996-12-20T01:39:57Z""#);
|
||||
}
|
||||
}
|
||||
|
||||
mod not_safe {
|
||||
use std::ops::Deref;
|
||||
|
||||
union Ref<'a, T: ?Sized> {
|
||||
shared: &'a T,
|
||||
uniq: &'a mut T,
|
||||
}
|
||||
|
||||
pub struct MaybeOwnedString<'a> {
|
||||
string: Ref<'a, str>,
|
||||
capacity: usize,
|
||||
}
|
||||
|
||||
impl<'a> MaybeOwnedString<'a> {
|
||||
pub fn from_str(string: &'a str) -> MaybeOwnedString<'a> {
|
||||
MaybeOwnedString {
|
||||
string: Ref { shared: string },
|
||||
capacity: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_string(mut string: String) -> MaybeOwnedString<'static> {
|
||||
string.shrink_to_fit(); // <= actually important for safety, todo use the Vec .as_ptr instead
|
||||
|
||||
let mut s = std::mem::ManuallyDrop::new(string);
|
||||
let ptr = s.as_mut_ptr();
|
||||
let len = s.len();
|
||||
let capacity = s.capacity();
|
||||
|
||||
let string = unsafe {
|
||||
std::str::from_utf8_unchecked_mut(std::slice::from_raw_parts_mut(ptr, len))
|
||||
};
|
||||
MaybeOwnedString {
|
||||
string: Ref { uniq: string },
|
||||
capacity,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn into_string(mut self) -> String {
|
||||
if self.capacity != 0 {
|
||||
let string = unsafe { &mut self.string.uniq };
|
||||
unsafe {
|
||||
return String::from_raw_parts(string.as_mut_ptr(), self.len(), self.capacity);
|
||||
};
|
||||
}
|
||||
self.deref().to_owned()
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
self.deref()
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Deref for MaybeOwnedString<'a> {
|
||||
type Target = str;
|
||||
|
||||
#[inline]
|
||||
fn deref(&self) -> &str {
|
||||
unsafe { self.string.shared }
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Drop for MaybeOwnedString<'a> {
|
||||
fn drop(&mut self) {
|
||||
// if capacity is 0, either it's an empty String so there is no dealloc to do, or it's
|
||||
// borrowed
|
||||
if self.capacity != 0 {
|
||||
let string = unsafe { &mut self.string.uniq };
|
||||
unsafe { String::from_raw_parts(string.as_mut_ptr(), self.len(), self.capacity) };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Clone for MaybeOwnedString<'a> {
|
||||
fn clone(&self) -> Self {
|
||||
if self.capacity == 0 {
|
||||
MaybeOwnedString {
|
||||
string: Ref {
|
||||
shared: unsafe { self.string.shared },
|
||||
},
|
||||
capacity: 0,
|
||||
}
|
||||
} else {
|
||||
MaybeOwnedString::from_string(self.deref().to_owned())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> std::fmt::Debug for MaybeOwnedString<'a> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(self.deref())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> PartialEq for MaybeOwnedString<'a> {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.deref() == other.deref()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use std::io;
|
||||
use std::ops::Range;
|
||||
|
||||
use common::{read_u32_vint, VInt};
|
||||
use common::VInt;
|
||||
|
||||
use crate::store::index::{Checkpoint, CHECKPOINT_PERIOD};
|
||||
use crate::DocId;
|
||||
@@ -85,15 +85,15 @@ impl CheckpointBlock {
|
||||
return Err(io::Error::new(io::ErrorKind::UnexpectedEof, ""));
|
||||
}
|
||||
self.checkpoints.clear();
|
||||
let len = read_u32_vint(data);
|
||||
let len = VInt::deserialize_u64(data)? as usize;
|
||||
if len == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let mut doc = read_u32_vint(data);
|
||||
let mut start_offset = read_u32_vint(data) as usize;
|
||||
let mut doc = VInt::deserialize_u64(data)? as DocId;
|
||||
let mut start_offset = VInt::deserialize_u64(data)? as usize;
|
||||
for _ in 0..len {
|
||||
let num_docs = read_u32_vint(data);
|
||||
let block_num_bytes = read_u32_vint(data) as usize;
|
||||
let num_docs = VInt::deserialize_u64(data)? as DocId;
|
||||
let block_num_bytes = VInt::deserialize_u64(data)? as usize;
|
||||
self.checkpoints.push(Checkpoint {
|
||||
doc_range: doc..doc + num_docs,
|
||||
byte_range: start_offset..start_offset + block_num_bytes,
|
||||
|
||||
@@ -96,7 +96,7 @@ pub mod tests {
|
||||
let mut doc = Document::default();
|
||||
doc.add_field_value(field_body, LOREM.to_string());
|
||||
doc.add_field_value(field_title, format!("Doc {i}"));
|
||||
store_writer.store(&doc, &schema).unwrap();
|
||||
store_writer.store(&doc).unwrap();
|
||||
}
|
||||
store_writer.close().unwrap();
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
use std::io;
|
||||
use std::iter::Sum;
|
||||
use std::ops::{AddAssign, Range};
|
||||
use std::ops::AddAssign;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use common::{BinarySerializable, HasLen};
|
||||
use common::{BinarySerializable, HasLen, VInt};
|
||||
use lru::LruCache;
|
||||
use ownedbytes::OwnedBytes;
|
||||
|
||||
@@ -140,10 +140,10 @@ impl StoreReader {
|
||||
self.cache.stats()
|
||||
}
|
||||
|
||||
/// Get checkpoint for `DocId`. The checkpoint can be used to load a block containing the
|
||||
/// Get checkpoint for DocId. The checkpoint can be used to load a block containing the
|
||||
/// document.
|
||||
///
|
||||
/// Advanced API. In most cases use [`get`](Self::get).
|
||||
/// Advanced API. In most cases use [get](Self::get).
|
||||
fn block_checkpoint(&self, doc_id: DocId) -> crate::Result<Checkpoint> {
|
||||
self.skip_index.seek(doc_id).ok_or_else(|| {
|
||||
crate::TantivyError::InvalidArgument(format!("Failed to lookup Doc #{}.", doc_id))
|
||||
@@ -160,7 +160,7 @@ impl StoreReader {
|
||||
|
||||
/// Loads and decompresses a block.
|
||||
///
|
||||
/// Advanced API. In most cases use [`get`](Self::get).
|
||||
/// Advanced API. In most cases use [get](Self::get).
|
||||
fn read_block(&self, checkpoint: &Checkpoint) -> io::Result<Block> {
|
||||
let cache_key = checkpoint.byte_range.start;
|
||||
if let Some(block) = self.cache.get_from_cache(cache_key) {
|
||||
@@ -205,21 +205,28 @@ impl StoreReader {
|
||||
|
||||
/// Advanced API.
|
||||
///
|
||||
/// In most cases use [`get_document_bytes`](Self::get_document_bytes).
|
||||
/// In most cases use [get_document_bytes](Self::get_document_bytes).
|
||||
fn get_document_bytes_from_block(
|
||||
block: OwnedBytes,
|
||||
doc_id: DocId,
|
||||
checkpoint: &Checkpoint,
|
||||
) -> crate::Result<OwnedBytes> {
|
||||
let doc_pos = doc_id - checkpoint.doc_range.start;
|
||||
let mut cursor = &block[..];
|
||||
let cursor_len_before = cursor.len();
|
||||
for _ in checkpoint.doc_range.start..doc_id {
|
||||
let doc_length = VInt::deserialize(&mut cursor)?.val() as usize;
|
||||
cursor = &cursor[doc_length..];
|
||||
}
|
||||
|
||||
let range = block_read_index(&block, doc_pos)?;
|
||||
Ok(block.slice(range))
|
||||
let doc_length = VInt::deserialize(&mut cursor)?.val() as usize;
|
||||
let start_pos = cursor_len_before - cursor.len();
|
||||
let end_pos = cursor_len_before - cursor.len() + doc_length;
|
||||
Ok(block.slice(start_pos..end_pos))
|
||||
}
|
||||
|
||||
/// Iterator over all Documents in their order as they are stored in the doc store.
|
||||
/// Use this, if you want to extract all Documents from the doc store.
|
||||
/// The `alive_bitset` has to be forwarded from the `SegmentReader` or the results may be wrong.
|
||||
/// The alive_bitset has to be forwarded from the `SegmentReader` or the results maybe wrong.
|
||||
pub fn iter<'a: 'b, 'b>(
|
||||
&'b self,
|
||||
alive_bitset: Option<&'a AliveBitSet>,
|
||||
@@ -230,9 +237,9 @@ impl StoreReader {
|
||||
})
|
||||
}
|
||||
|
||||
/// Iterator over all raw Documents in their order as they are stored in the doc store.
|
||||
/// Iterator over all RawDocuments in their order as they are stored in the doc store.
|
||||
/// Use this, if you want to extract all Documents from the doc store.
|
||||
/// The `alive_bitset` has to be forwarded from the `SegmentReader` or the results may be wrong.
|
||||
/// The alive_bitset has to be forwarded from the `SegmentReader` or the results maybe wrong.
|
||||
pub(crate) fn iter_raw<'a: 'b, 'b>(
|
||||
&'b self,
|
||||
alive_bitset: Option<&'a AliveBitSet>,
|
||||
@@ -247,7 +254,9 @@ impl StoreReader {
|
||||
let mut curr_block = curr_checkpoint
|
||||
.as_ref()
|
||||
.map(|checkpoint| self.read_block(checkpoint).map_err(|e| e.kind())); // map error in order to enable cloning
|
||||
let mut doc_pos = 0;
|
||||
let mut block_start_pos = 0;
|
||||
let mut num_skipped = 0;
|
||||
let mut reset_block_pos = false;
|
||||
(0..last_doc_id)
|
||||
.filter_map(move |doc_id| {
|
||||
// filter_map is only used to resolve lifetime issues between the two closures on
|
||||
@@ -259,19 +268,24 @@ impl StoreReader {
|
||||
curr_block = curr_checkpoint
|
||||
.as_ref()
|
||||
.map(|checkpoint| self.read_block(checkpoint).map_err(|e| e.kind()));
|
||||
doc_pos = 0;
|
||||
reset_block_pos = true;
|
||||
num_skipped = 0;
|
||||
}
|
||||
|
||||
let alive = alive_bitset.map_or(true, |bitset| bitset.is_alive(doc_id));
|
||||
let res = if alive {
|
||||
Some((curr_block.clone(), doc_pos))
|
||||
if alive {
|
||||
let ret = Some((curr_block.clone(), num_skipped, reset_block_pos));
|
||||
// the map block will move over the num_skipped, so we reset to 0
|
||||
num_skipped = 0;
|
||||
reset_block_pos = false;
|
||||
ret
|
||||
} else {
|
||||
// we keep the number of skipped documents to move forward in the map block
|
||||
num_skipped += 1;
|
||||
None
|
||||
};
|
||||
doc_pos += 1;
|
||||
res
|
||||
}
|
||||
})
|
||||
.map(move |(block, doc_pos)| {
|
||||
.map(move |(block, num_skipped, reset_block_pos)| {
|
||||
let block = block
|
||||
.ok_or_else(|| {
|
||||
DataCorruption::comment_only(
|
||||
@@ -282,9 +296,30 @@ impl StoreReader {
|
||||
.map_err(|error_kind| {
|
||||
std::io::Error::new(error_kind, "error when reading block in doc store")
|
||||
})?;
|
||||
// this flag is set, when filter_map moved to the next block
|
||||
if reset_block_pos {
|
||||
block_start_pos = 0;
|
||||
}
|
||||
let mut cursor = &block[block_start_pos..];
|
||||
let mut pos = 0;
|
||||
// move forward 1 doc + num_skipped in block and return length of current doc
|
||||
let doc_length = loop {
|
||||
let doc_length = VInt::deserialize(&mut cursor)?.val() as usize;
|
||||
let num_bytes_read = block[block_start_pos..].len() - cursor.len();
|
||||
block_start_pos += num_bytes_read;
|
||||
|
||||
let range = block_read_index(&block, doc_pos)?;
|
||||
Ok(block.slice(range))
|
||||
pos += 1;
|
||||
if pos == num_skipped + 1 {
|
||||
break doc_length;
|
||||
} else {
|
||||
block_start_pos += doc_length;
|
||||
cursor = &block[block_start_pos..];
|
||||
}
|
||||
};
|
||||
let end_pos = block_start_pos + doc_length;
|
||||
let doc_bytes = block.slice(block_start_pos..end_pos);
|
||||
block_start_pos = end_pos;
|
||||
Ok(doc_bytes)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -294,33 +329,11 @@ impl StoreReader {
|
||||
}
|
||||
}
|
||||
|
||||
fn block_read_index(block: &[u8], doc_pos: u32) -> crate::Result<Range<usize>> {
|
||||
let doc_pos = doc_pos as usize;
|
||||
let size_of_u32 = std::mem::size_of::<u32>();
|
||||
|
||||
let index_len_pos = block.len() - size_of_u32;
|
||||
let index_len = u32::deserialize(&mut &block[index_len_pos..])? as usize;
|
||||
|
||||
if doc_pos > index_len {
|
||||
return Err(crate::TantivyError::InternalError(
|
||||
"Attempted to read doc from wrong block".to_owned(),
|
||||
));
|
||||
}
|
||||
|
||||
let index_start = block.len() - (index_len + 1) * size_of_u32;
|
||||
let index = &block[index_start..index_start + index_len * size_of_u32];
|
||||
|
||||
let start_offset = u32::deserialize(&mut &index[doc_pos * size_of_u32..])? as usize;
|
||||
let end_offset = u32::deserialize(&mut &index[(doc_pos + 1) * size_of_u32..])
|
||||
.unwrap_or(index_start as u32) as usize;
|
||||
Ok(start_offset..end_offset)
|
||||
}
|
||||
|
||||
#[cfg(feature = "quickwit")]
|
||||
impl StoreReader {
|
||||
/// Advanced API.
|
||||
///
|
||||
/// In most cases use [`get_async`](Self::get_async)
|
||||
/// In most cases use [get_async](Self::get_async)
|
||||
///
|
||||
/// Loads and decompresses a block asynchronously.
|
||||
async fn read_block_async(&self, checkpoint: &Checkpoint) -> crate::AsyncIoResult<Block> {
|
||||
@@ -344,14 +357,14 @@ impl StoreReader {
|
||||
Ok(decompressed_block)
|
||||
}
|
||||
|
||||
/// Reads raw bytes of a given document asynchronously.
|
||||
/// Fetches a document asynchronously.
|
||||
pub async fn get_document_bytes_async(&self, doc_id: DocId) -> crate::Result<OwnedBytes> {
|
||||
let checkpoint = self.block_checkpoint(doc_id)?;
|
||||
let block = self.read_block_async(&checkpoint).await?;
|
||||
Self::get_document_bytes_from_block(block, doc_id, &checkpoint)
|
||||
}
|
||||
|
||||
/// Fetches a document asynchronously. Async version of [`get`](Self::get).
|
||||
/// Reads raw bytes of a given document. Async version of [get](Self::get).
|
||||
pub async fn get_async(&self, doc_id: DocId) -> crate::Result<Document> {
|
||||
let mut doc_bytes = self.get_document_bytes_async(doc_id).await?;
|
||||
Ok(Document::deserialize(&mut doc_bytes)?)
|
||||
@@ -414,7 +427,7 @@ mod tests {
|
||||
assert_eq!(store.cache_stats().cache_hits, 1);
|
||||
assert_eq!(store.cache_stats().cache_misses, 2);
|
||||
|
||||
assert_eq!(store.cache.peek_lru(), Some(11163));
|
||||
assert_eq!(store.cache.peek_lru(), Some(9210));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
use std::io;
|
||||
use std::io::{self, Write};
|
||||
|
||||
use common::BinarySerializable;
|
||||
use common::{BinarySerializable, VInt};
|
||||
|
||||
use super::compressors::Compressor;
|
||||
use super::StoreReader;
|
||||
use crate::directory::WritePtr;
|
||||
use crate::schema::{Document, Schema};
|
||||
use crate::schema::Document;
|
||||
use crate::store::store_compressor::BlockCompressor;
|
||||
use crate::DocId;
|
||||
|
||||
@@ -20,8 +20,8 @@ pub struct StoreWriter {
|
||||
compressor: Compressor,
|
||||
block_size: usize,
|
||||
num_docs_in_current_block: DocId,
|
||||
intermediary_buffer: Vec<u8>,
|
||||
current_block: Vec<u8>,
|
||||
doc_pos: Vec<u32>,
|
||||
block_compressor: BlockCompressor,
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ impl StoreWriter {
|
||||
compressor,
|
||||
block_size,
|
||||
num_docs_in_current_block: 0,
|
||||
doc_pos: Vec::new(),
|
||||
intermediary_buffer: Vec::new(),
|
||||
current_block: Vec::new(),
|
||||
block_compressor,
|
||||
})
|
||||
@@ -53,15 +53,12 @@ impl StoreWriter {
|
||||
|
||||
/// The memory used (inclusive childs)
|
||||
pub fn mem_usage(&self) -> usize {
|
||||
self.current_block.capacity() + self.doc_pos.capacity() * std::mem::size_of::<u32>()
|
||||
self.intermediary_buffer.capacity() + self.current_block.capacity()
|
||||
}
|
||||
|
||||
/// Checks if the current block is full, and if so, compresses and flushes it.
|
||||
fn check_flush_block(&mut self) -> io::Result<()> {
|
||||
// this does not count the VInt storing the index lenght itself, but it is negligible in
|
||||
// front of everything else.
|
||||
let index_len = self.doc_pos.len() * std::mem::size_of::<usize>();
|
||||
if self.current_block.len() + index_len > self.block_size {
|
||||
if self.current_block.len() > self.block_size {
|
||||
self.send_current_block_to_compressor()?;
|
||||
}
|
||||
Ok(())
|
||||
@@ -73,19 +70,8 @@ impl StoreWriter {
|
||||
if self.current_block.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let size_of_u32 = std::mem::size_of::<u32>();
|
||||
self.current_block
|
||||
.reserve((self.doc_pos.len() + 1) * size_of_u32);
|
||||
|
||||
for pos in self.doc_pos.iter() {
|
||||
pos.serialize(&mut self.current_block)?;
|
||||
}
|
||||
(self.doc_pos.len() as u32).serialize(&mut self.current_block)?;
|
||||
|
||||
self.block_compressor
|
||||
.compress_block_and_write(&self.current_block, self.num_docs_in_current_block)?;
|
||||
self.doc_pos.clear();
|
||||
self.current_block.clear();
|
||||
self.num_docs_in_current_block = 0;
|
||||
Ok(())
|
||||
@@ -95,9 +81,16 @@ impl StoreWriter {
|
||||
///
|
||||
/// The document id is implicitly the current number
|
||||
/// of documents.
|
||||
pub fn store(&mut self, document: &Document, schema: &Schema) -> io::Result<()> {
|
||||
self.doc_pos.push(self.current_block.len() as u32);
|
||||
document.serialize_stored(schema, &mut self.current_block)?;
|
||||
pub fn store(&mut self, stored_document: &Document) -> io::Result<()> {
|
||||
self.intermediary_buffer.clear();
|
||||
stored_document.serialize(&mut self.intermediary_buffer)?;
|
||||
// calling store bytes would be preferable for code reuse, but then we can't use
|
||||
// intermediary_buffer due to the borrow checker
|
||||
// a new buffer costs ~1% indexing performance
|
||||
let doc_num_bytes = self.intermediary_buffer.len();
|
||||
VInt(doc_num_bytes as u64).serialize_into_vec(&mut self.current_block);
|
||||
self.current_block
|
||||
.write_all(&self.intermediary_buffer[..])?;
|
||||
self.num_docs_in_current_block += 1;
|
||||
self.check_flush_block()?;
|
||||
Ok(())
|
||||
@@ -108,7 +101,8 @@ impl StoreWriter {
|
||||
/// The document id is implicitly the current number
|
||||
/// of documents.
|
||||
pub fn store_bytes(&mut self, serialized_document: &[u8]) -> io::Result<()> {
|
||||
self.doc_pos.push(self.current_block.len() as u32);
|
||||
let doc_num_bytes = serialized_document.len();
|
||||
VInt(doc_num_bytes as u64).serialize_into_vec(&mut self.current_block);
|
||||
self.current_block.extend_from_slice(serialized_document);
|
||||
self.num_docs_in_current_block += 1;
|
||||
self.check_flush_block()?;
|
||||
|
||||
@@ -13,7 +13,7 @@ pub struct SSTableIndex {
|
||||
|
||||
impl SSTableIndex {
|
||||
pub(crate) fn load(data: &[u8]) -> Result<SSTableIndex, DataCorruption> {
|
||||
ciborium::de::from_reader(data)
|
||||
serde_cbor::de::from_slice(data)
|
||||
.map_err(|_| DataCorruption::comment_only("SSTable index is corrupted"))
|
||||
}
|
||||
|
||||
@@ -85,9 +85,9 @@ impl SSTableIndexBuilder {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn serialize<W: std::io::Write>(&self, wrt: W) -> io::Result<()> {
|
||||
ciborium::ser::into_writer(&self.index, wrt)
|
||||
.map_err(|err| io::Error::new(io::ErrorKind::Other, err))
|
||||
pub fn serialize(&self, wrt: &mut dyn io::Write) -> io::Result<()> {
|
||||
serde_cbor::ser::to_writer(wrt, &self.index).unwrap();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,8 +24,6 @@ impl SSTable for TermInfoSSTable {
|
||||
type Reader = TermInfoReader;
|
||||
type Writer = TermInfoWriter;
|
||||
}
|
||||
|
||||
/// Builder for the new term dictionary.
|
||||
pub struct TermDictionaryBuilder<W: io::Write> {
|
||||
sstable_writer: Writer<W, TermInfoWriter>,
|
||||
}
|
||||
@@ -140,7 +138,6 @@ impl TermDictionary {
|
||||
})
|
||||
}
|
||||
|
||||
/// Creates a term dictionary from the supplied bytes.
|
||||
pub fn from_bytes(owned_bytes: OwnedBytes) -> crate::Result<TermDictionary> {
|
||||
TermDictionary::open(FileSlice::new(Arc::new(owned_bytes)))
|
||||
}
|
||||
@@ -232,19 +229,19 @@ impl TermDictionary {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Returns a range builder, to stream all of the terms
|
||||
/// within an interval.
|
||||
// Returns a range builder, to stream all of the terms
|
||||
// within an interval.
|
||||
pub fn range(&self) -> TermStreamerBuilder<'_> {
|
||||
TermStreamerBuilder::new(self, AlwaysMatch)
|
||||
}
|
||||
|
||||
/// A stream of all the sorted terms.
|
||||
// A stream of all the sorted terms.
|
||||
pub fn stream(&self) -> io::Result<TermStreamer<'_>> {
|
||||
self.range().into_stream()
|
||||
}
|
||||
|
||||
/// Returns a search builder, to stream all of the terms
|
||||
/// within the Automaton
|
||||
// Returns a search builder, to stream all of the terms
|
||||
// within the Automaton
|
||||
pub fn search<'a, A: Automaton + 'a>(&'a self, automaton: A) -> TermStreamerBuilder<'a, A>
|
||||
where A::State: Clone {
|
||||
TermStreamerBuilder::<A>::new(self, automaton)
|
||||
|
||||
@@ -126,7 +126,6 @@ mod ngram_tokenizer;
|
||||
mod raw_tokenizer;
|
||||
mod remove_long;
|
||||
mod simple_tokenizer;
|
||||
mod split_compound_words;
|
||||
mod stemmer;
|
||||
mod stop_word_filter;
|
||||
mod tokenized_string;
|
||||
@@ -142,7 +141,6 @@ pub use self::ngram_tokenizer::NgramTokenizer;
|
||||
pub use self::raw_tokenizer::RawTokenizer;
|
||||
pub use self::remove_long::RemoveLongFilter;
|
||||
pub use self::simple_tokenizer::SimpleTokenizer;
|
||||
pub use self::split_compound_words::SplitCompoundWords;
|
||||
pub use self::stemmer::{Language, Stemmer};
|
||||
pub use self::stop_word_filter::StopWordFilter;
|
||||
pub use self::tokenized_string::{PreTokenizedStream, PreTokenizedString};
|
||||
|
||||
@@ -1,252 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use aho_corasick::{AhoCorasick, AhoCorasickBuilder, MatchKind, StateID};
|
||||
|
||||
use super::{BoxTokenStream, Token, TokenFilter, TokenStream};
|
||||
|
||||
/// A [`TokenFilter`] which splits compound words into their parts
|
||||
/// based on a given dictionary.
|
||||
///
|
||||
/// Words only will be split if they can be fully decomposed into
|
||||
/// consecutive matches into the given dictionary.
|
||||
///
|
||||
/// This is mostly useful to split [compound nouns][compound] common to many
|
||||
/// Germanic languages into their constituents.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// The quality of the dictionary determines the quality of the splits,
|
||||
/// e.g. the missing stem "back" of "backen" implies that "brotbackautomat"
|
||||
/// is not split in the following example.
|
||||
///
|
||||
/// ```rust
|
||||
/// use tantivy::tokenizer::{SimpleTokenizer, SplitCompoundWords, TextAnalyzer};
|
||||
///
|
||||
/// let tokenizer =
|
||||
/// TextAnalyzer::from(SimpleTokenizer).filter(SplitCompoundWords::from_dictionary([
|
||||
/// "dampf", "schiff", "fahrt", "brot", "backen", "automat",
|
||||
/// ]));
|
||||
///
|
||||
/// let mut stream = tokenizer.token_stream("dampfschifffahrt");
|
||||
/// assert_eq!(stream.next().unwrap().text, "dampf");
|
||||
/// assert_eq!(stream.next().unwrap().text, "schiff");
|
||||
/// assert_eq!(stream.next().unwrap().text, "fahrt");
|
||||
/// assert_eq!(stream.next(), None);
|
||||
///
|
||||
/// let mut stream = tokenizer.token_stream("brotbackautomat");
|
||||
/// assert_eq!(stream.next().unwrap().text, "brotbackautomat");
|
||||
/// assert_eq!(stream.next(), None);
|
||||
/// ```
|
||||
///
|
||||
/// [compound]: https://en.wikipedia.org/wiki/Compound_(linguistics)
|
||||
#[derive(Clone)]
|
||||
pub struct SplitCompoundWords<S: StateID> {
|
||||
dict: Arc<AhoCorasick<S>>,
|
||||
}
|
||||
|
||||
impl SplitCompoundWords<usize> {
|
||||
/// Create a filter from a given dictionary.
|
||||
///
|
||||
/// The dictionary will be used to construct an [`AhoCorasick`] automaton
|
||||
/// with reasonable defaults. See [`from_automaton`][Self::from_automaton] if
|
||||
/// more control over its construction is required.
|
||||
pub fn from_dictionary<I, P>(dict: I) -> Self
|
||||
where
|
||||
I: IntoIterator<Item = P>,
|
||||
P: AsRef<[u8]>,
|
||||
{
|
||||
let dict = AhoCorasickBuilder::new()
|
||||
.match_kind(MatchKind::LeftmostLongest)
|
||||
.build(dict);
|
||||
|
||||
Self::from_automaton(dict)
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: StateID> SplitCompoundWords<S> {
|
||||
/// Create a filter from a given automaton.
|
||||
///
|
||||
/// The automaton should use one of the leftmost-first match kinds
|
||||
/// and it should not be anchored.
|
||||
pub fn from_automaton(dict: AhoCorasick<S>) -> Self {
|
||||
Self {
|
||||
dict: Arc::new(dict),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: StateID + Send + Sync + 'static> TokenFilter for SplitCompoundWords<S> {
|
||||
fn transform<'a>(&self, stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> {
|
||||
BoxTokenStream::from(SplitCompoundWordsTokenStream {
|
||||
dict: self.dict.clone(),
|
||||
tail: stream,
|
||||
cuts: Vec::new(),
|
||||
parts: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct SplitCompoundWordsTokenStream<'a, S: StateID> {
|
||||
dict: Arc<AhoCorasick<S>>,
|
||||
tail: BoxTokenStream<'a>,
|
||||
cuts: Vec<usize>,
|
||||
parts: Vec<Token>,
|
||||
}
|
||||
|
||||
impl<'a, S: StateID> SplitCompoundWordsTokenStream<'a, S> {
|
||||
// Will use `self.cuts` to fill `self.parts` if `self.tail.token()`
|
||||
// can fully be split into consecutive matches against `self.dict`.
|
||||
fn split(&mut self) {
|
||||
let token = self.tail.token();
|
||||
let mut text = token.text.as_str();
|
||||
|
||||
self.cuts.clear();
|
||||
let mut pos = 0;
|
||||
|
||||
for match_ in self.dict.find_iter(text) {
|
||||
if pos != match_.start() {
|
||||
break;
|
||||
}
|
||||
|
||||
self.cuts.push(pos);
|
||||
pos = match_.end();
|
||||
}
|
||||
|
||||
if pos == token.text.len() {
|
||||
// Fill `self.parts` in reverse order,
|
||||
// so that `self.parts.pop()` yields
|
||||
// the tokens in their original order.
|
||||
for pos in self.cuts.iter().rev() {
|
||||
let (head, tail) = text.split_at(*pos);
|
||||
|
||||
text = head;
|
||||
self.parts.push(Token {
|
||||
text: tail.to_owned(),
|
||||
..*token
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, S: StateID> TokenStream for SplitCompoundWordsTokenStream<'a, S> {
|
||||
fn advance(&mut self) -> bool {
|
||||
self.parts.pop();
|
||||
|
||||
if !self.parts.is_empty() {
|
||||
return true;
|
||||
}
|
||||
|
||||
if !self.tail.advance() {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Will yield either `self.parts.last()` or
|
||||
// `self.tail.token()` if it could not be split.
|
||||
self.split();
|
||||
true
|
||||
}
|
||||
|
||||
fn token(&self) -> &Token {
|
||||
self.parts.last().unwrap_or_else(|| self.tail.token())
|
||||
}
|
||||
|
||||
fn token_mut(&mut self) -> &mut Token {
|
||||
self.parts
|
||||
.last_mut()
|
||||
.unwrap_or_else(|| self.tail.token_mut())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::tokenizer::{SimpleTokenizer, TextAnalyzer};
|
||||
|
||||
#[test]
|
||||
fn splitting_compound_words_works() {
|
||||
let tokenizer = TextAnalyzer::from(SimpleTokenizer)
|
||||
.filter(SplitCompoundWords::from_dictionary(["foo", "bar"]));
|
||||
|
||||
{
|
||||
let mut stream = tokenizer.token_stream("");
|
||||
assert_eq!(stream.next(), None);
|
||||
}
|
||||
|
||||
{
|
||||
let mut stream = tokenizer.token_stream("foo bar");
|
||||
assert_eq!(stream.next().unwrap().text, "foo");
|
||||
assert_eq!(stream.next().unwrap().text, "bar");
|
||||
assert_eq!(stream.next(), None);
|
||||
}
|
||||
|
||||
{
|
||||
let mut stream = tokenizer.token_stream("foobar");
|
||||
assert_eq!(stream.next().unwrap().text, "foo");
|
||||
assert_eq!(stream.next().unwrap().text, "bar");
|
||||
assert_eq!(stream.next(), None);
|
||||
}
|
||||
|
||||
{
|
||||
let mut stream = tokenizer.token_stream("foobarbaz");
|
||||
assert_eq!(stream.next().unwrap().text, "foobarbaz");
|
||||
assert_eq!(stream.next(), None);
|
||||
}
|
||||
|
||||
{
|
||||
let mut stream = tokenizer.token_stream("baz foobar qux");
|
||||
assert_eq!(stream.next().unwrap().text, "baz");
|
||||
assert_eq!(stream.next().unwrap().text, "foo");
|
||||
assert_eq!(stream.next().unwrap().text, "bar");
|
||||
assert_eq!(stream.next().unwrap().text, "qux");
|
||||
assert_eq!(stream.next(), None);
|
||||
}
|
||||
|
||||
{
|
||||
let mut stream = tokenizer.token_stream("foobar foobar");
|
||||
assert_eq!(stream.next().unwrap().text, "foo");
|
||||
assert_eq!(stream.next().unwrap().text, "bar");
|
||||
assert_eq!(stream.next().unwrap().text, "foo");
|
||||
assert_eq!(stream.next().unwrap().text, "bar");
|
||||
assert_eq!(stream.next(), None);
|
||||
}
|
||||
|
||||
{
|
||||
let mut stream = tokenizer.token_stream("foobar foo bar foobar");
|
||||
assert_eq!(stream.next().unwrap().text, "foo");
|
||||
assert_eq!(stream.next().unwrap().text, "bar");
|
||||
assert_eq!(stream.next().unwrap().text, "foo");
|
||||
assert_eq!(stream.next().unwrap().text, "bar");
|
||||
assert_eq!(stream.next().unwrap().text, "foo");
|
||||
assert_eq!(stream.next().unwrap().text, "bar");
|
||||
assert_eq!(stream.next(), None);
|
||||
}
|
||||
|
||||
{
|
||||
let mut stream = tokenizer.token_stream("foobazbar foo bar foobar");
|
||||
assert_eq!(stream.next().unwrap().text, "foobazbar");
|
||||
assert_eq!(stream.next().unwrap().text, "foo");
|
||||
assert_eq!(stream.next().unwrap().text, "bar");
|
||||
assert_eq!(stream.next().unwrap().text, "foo");
|
||||
assert_eq!(stream.next().unwrap().text, "bar");
|
||||
assert_eq!(stream.next(), None);
|
||||
}
|
||||
|
||||
{
|
||||
let mut stream = tokenizer.token_stream("foobar qux foobar");
|
||||
assert_eq!(stream.next().unwrap().text, "foo");
|
||||
assert_eq!(stream.next().unwrap().text, "bar");
|
||||
assert_eq!(stream.next().unwrap().text, "qux");
|
||||
assert_eq!(stream.next().unwrap().text, "foo");
|
||||
assert_eq!(stream.next().unwrap().text, "bar");
|
||||
assert_eq!(stream.next(), None);
|
||||
}
|
||||
|
||||
{
|
||||
let mut stream = tokenizer.token_stream("barfoo");
|
||||
assert_eq!(stream.next().unwrap().text, "bar");
|
||||
assert_eq!(stream.next().unwrap().text, "foo");
|
||||
assert_eq!(stream.next(), None);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,3 @@
|
||||
use std::borrow::Cow;
|
||||
use std::mem;
|
||||
|
||||
use rust_stemmers::{self, Algorithm};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
@@ -87,7 +84,6 @@ impl TokenFilter for Stemmer {
|
||||
BoxTokenStream::from(StemmerTokenStream {
|
||||
tail: token_stream,
|
||||
stemmer: inner_stemmer,
|
||||
buffer: String::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -95,7 +91,6 @@ impl TokenFilter for Stemmer {
|
||||
pub struct StemmerTokenStream<'a> {
|
||||
tail: BoxTokenStream<'a>,
|
||||
stemmer: rust_stemmers::Stemmer,
|
||||
buffer: String,
|
||||
}
|
||||
|
||||
impl<'a> TokenStream for StemmerTokenStream<'a> {
|
||||
@@ -103,16 +98,10 @@ impl<'a> TokenStream for StemmerTokenStream<'a> {
|
||||
if !self.tail.advance() {
|
||||
return false;
|
||||
}
|
||||
let token = self.tail.token_mut();
|
||||
let stemmed_str = self.stemmer.stem(&token.text);
|
||||
match stemmed_str {
|
||||
Cow::Owned(stemmed_str) => token.text = stemmed_str,
|
||||
Cow::Borrowed(stemmed_str) => {
|
||||
self.buffer.clear();
|
||||
self.buffer.push_str(stemmed_str);
|
||||
mem::swap(&mut token.text, &mut self.buffer);
|
||||
}
|
||||
}
|
||||
// TODO remove allocation
|
||||
let stemmed_str: String = self.stemmer.stem(&self.token().text).into_owned();
|
||||
self.token_mut().text.clear();
|
||||
self.token_mut().text.push_str(&stemmed_str);
|
||||
true
|
||||
}
|
||||
|
||||
|
||||
@@ -10,21 +10,28 @@
|
||||
//! assert_eq!(stream.next().unwrap().text, "crafty");
|
||||
//! assert!(stream.next().is_none());
|
||||
//! ```
|
||||
use rustc_hash::FxHashSet;
|
||||
use std::collections::HashSet;
|
||||
use std::hash::BuildHasherDefault;
|
||||
|
||||
use fnv::FnvHasher;
|
||||
|
||||
use super::{Token, TokenFilter, TokenStream};
|
||||
use crate::tokenizer::BoxTokenStream;
|
||||
|
||||
// configure our hashers for SPEED
|
||||
type StopWordHasher = BuildHasherDefault<FnvHasher>;
|
||||
type StopWordHashSet = HashSet<String, StopWordHasher>;
|
||||
|
||||
/// `TokenFilter` that removes stop words from a token stream
|
||||
#[derive(Clone)]
|
||||
pub struct StopWordFilter {
|
||||
words: FxHashSet<String>,
|
||||
words: StopWordHashSet,
|
||||
}
|
||||
|
||||
impl StopWordFilter {
|
||||
/// Creates a `StopWordFilter` given a list of words to remove
|
||||
pub fn remove(words: Vec<String>) -> StopWordFilter {
|
||||
let mut set = FxHashSet::default();
|
||||
let mut set = StopWordHashSet::default();
|
||||
|
||||
for word in words {
|
||||
set.insert(word);
|
||||
@@ -45,7 +52,7 @@ impl StopWordFilter {
|
||||
}
|
||||
|
||||
pub struct StopWordFilterStream<'a> {
|
||||
words: FxHashSet<String>,
|
||||
words: StopWordHashSet,
|
||||
tail: BoxTokenStream<'a>,
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user