rust/tantivy
1
0
Fork 0
mirror of https://github.com/quickwit-oss/tantivy.git synced 2026-01-06 01:02:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

half way through removing FastFieldsReader

Browse Source
This commit is contained in:
Paul Masurel
2017-08-13 18:39:45 +09:00
parent 413d0e1719
commit 8d466b8a76
5 changed files with 63 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/core/searcher.rs
View File

@@ -23,7 +23,6 @@ pub struct Searcher {
segment_readers: Vec<SegmentReader>,
}
impl Searcher {
/// Fetches a document from tantivy's store given a `DocAddress`.
///

40
src/core/segment_reader.rs
View File

@@ -17,7 +17,7 @@ use common::CompositeFile;
use std::fmt;
use core::FieldReader;
use schema::Field;
use fastfield::{FastFieldsReader, FastFieldReader, U64FastFieldReader};
use fastfield::{FastFieldReader, U64FastFieldReader};
use schema::Schema;
@@ -43,10 +43,10 @@ pub struct SegmentReader {
termdict_composite: CompositeFile,
postings_composite: CompositeFile,
positions_composite: CompositeFile,
fast_fields_composite: CompositeFile,
fieldnorms_composite: CompositeFile,
store_reader: StoreReader,
fast_fields_reader: Arc<FastFieldsReader>,
fieldnorms_reader: Arc<FastFieldsReader>,
delete_bitset: DeleteBitSet,
schema: Schema,
}
@@ -75,11 +75,6 @@ impl SegmentReader {
self.delete_bitset.len() as DocId
}
#[doc(hidden)]
pub fn fast_fields_reader(&self) -> &FastFieldsReader {
&*self.fast_fields_reader
}
/// Accessor to a segment's fast field reader given a field.
///
/// Returns the u64 fast value reader if the field
@@ -91,16 +86,17 @@ impl SegmentReader {
/// # Panics
/// May panic if the index is corrupted.
pub fn get_fast_field_reader<TFastFieldReader: FastFieldReader>
(&self,
field: Field)
-> fastfield::Result<TFastFieldReader> {
(&self, field: Field) -> fastfield::Result<TFastFieldReader> {
let field_entry = self.schema.get_field_entry(field);
if !TFastFieldReader::is_enabled(field_entry.field_type()) {
Err(FastFieldNotAvailableError::new(field_entry))
} else {
Ok(self.fast_fields_reader
.open_reader(field)
.expect("Fast field file corrupted."))
self.fast_fields_composite
.open_read(field)
.ok_or_else(|| {
FastFieldNotAvailableError::new(field_entry)
})
.map(TFastFieldReader::open)
}
}
@@ -113,7 +109,9 @@ impl SegmentReader {
/// They are simply stored as a fast field, serialized in
/// the `.fieldnorm` file of the segment.
pub fn get_fieldnorms_reader(&self, field: Field) -> Option<U64FastFieldReader> {
self.fieldnorms_reader.open_reader(field)
self.fieldnorms_composite
.open_read(field)
.map(U64FastFieldReader::open)
}
/// Accessor to the segment's `StoreReader`.
@@ -143,11 +141,11 @@ impl SegmentReader {
};
let fast_field_data = segment.open_read(SegmentComponent::FASTFIELDS)?;
let fast_fields_reader = FastFieldsReader::from_source(fast_field_data)?;
let fast_fields_data = segment.open_read(SegmentComponent::FASTFIELDS)?;
let fast_fields_composite = CompositeFile::open(fast_fields_data)?;
let fieldnorms_data = segment.open_read(SegmentComponent::FIELDNORMS)?;
let fieldnorms_reader = FastFieldsReader::from_source(fieldnorms_data)?;
let fieldnorms_composite = CompositeFile::open(fieldnorms_data)?;
let delete_bitset = if segment.meta().has_deletes() {
@@ -161,12 +159,12 @@ impl SegmentReader {
Ok(SegmentReader {
field_reader_cache: Arc::new(RwLock::new(HashMap::new())),
segment_meta: segment.meta().clone(),
postings_composite: postings_composite,
termdict_composite: termdict_composite,
postings_composite: postings_composite,
fast_fields_composite: fast_fields_composite,
fieldnorms_composite: fieldnorms_composite,
segment_id: segment.id(),
store_reader: store_reader,
fast_fields_reader: Arc::new(fast_fields_reader),
fieldnorms_reader: Arc::new(fieldnorms_reader),
delete_bitset: delete_bitset,
positions_composite: positions_composite,
schema: schema,

45
src/fastfield/mod.rs
View File

@@ -32,7 +32,7 @@ mod delete;
pub use self::delete::write_delete_bitset;
pub use self::delete::DeleteBitSet;
pub use self::writer::{FastFieldsWriter, IntFastFieldWriter};
pub use self::reader::{FastFieldsReader, U64FastFieldReader, I64FastFieldReader};
pub use self::reader::{U64FastFieldReader, I64FastFieldReader};
pub use self::reader::FastFieldReader;
pub use self::serializer::FastFieldSerializer;
pub use self::error::{Result, FastFieldNotAvailableError};
@@ -51,6 +51,7 @@ mod tests {
use fastfield::FastFieldReader;
use rand::Rng;
use rand::SeedableRng;
use common::CompositeFile;
use rand::XorShiftRng;
lazy_static! {
@@ -96,6 +97,8 @@ mod tests {
{
assert_eq!(source.len(), 35 as usize);
}
// TODO uncomment
/*
{
let fast_field_readers = FastFieldsReader::from_source(source).unwrap();
let fast_field_reader: U64FastFieldReader =
@@ -104,6 +107,7 @@ mod tests {
assert_eq!(fast_field_reader.get(1), 14u64);
assert_eq!(fast_field_reader.get(2), 2u64);
}
*/
}
#[test]
@@ -131,9 +135,9 @@ mod tests {
assert_eq!(source.len(), 60 as usize);
}
{
let fast_field_readers = FastFieldsReader::from_source(source).unwrap();
let fast_fields_composite = CompositeFile::open(source).unwrap();
let fast_field_reader: U64FastFieldReader =
fast_field_readers.open_reader(*FIELD).unwrap();
U64FastFieldReader::open(fast_fields_composite.open_read(*FIELD).unwrap());
assert_eq!(fast_field_reader.get(0), 4u64);
assert_eq!(fast_field_reader.get(1), 14_082_001u64);
assert_eq!(fast_field_reader.get(2), 3_052u64);
@@ -167,9 +171,9 @@ mod tests {
assert_eq!(source.len(), 33 as usize);
}
{
let fast_field_readers = FastFieldsReader::from_source(source).unwrap();
let fast_fields_composite = CompositeFile::open(source).unwrap();
let fast_field_reader: U64FastFieldReader =
fast_field_readers.open_reader(*FIELD).unwrap();
U64FastFieldReader::open(fast_fields_composite.open_read(*FIELD).unwrap());
for doc in 0..10_000 {
assert_eq!(fast_field_reader.get(doc), 100_000u64);
}
@@ -200,9 +204,10 @@ mod tests {
assert_eq!(source.len(), 80041 as usize);
}
{
let fast_field_readers = FastFieldsReader::from_source(source).unwrap();
let fast_fields_composite = CompositeFile::open(source).unwrap();
let fast_field_reader: U64FastFieldReader =
fast_field_readers.open_reader(*FIELD).unwrap();
U64FastFieldReader::open(fast_fields_composite.open_read(*FIELD).unwrap());
assert_eq!(fast_field_reader.get(0), 0u64);
for doc in 1..10_001 {
assert_eq!(fast_field_reader.get(doc),
@@ -236,9 +241,10 @@ mod tests {
assert_eq!(source.len(), 17708 as usize);
}
{
let fast_field_readers = FastFieldsReader::from_source(source).unwrap();
let fast_fields_composite = CompositeFile::open(source).unwrap();
let fast_field_reader: I64FastFieldReader =
fast_field_readers.open_reader(i64_field).unwrap();
I64FastFieldReader::open(fast_fields_composite.open_read(i64_field).unwrap());
assert_eq!(fast_field_reader.min_value(), -100i64);
assert_eq!(fast_field_reader.max_value(), 9_999i64);
for (doc, i) in (-100i64..10_000i64).enumerate() {
@@ -272,9 +278,10 @@ mod tests {
let source = directory.open_read(&path).unwrap();
{
let fast_field_readers = FastFieldsReader::from_source(source).unwrap();
let fast_fields_composite = CompositeFile::open(source).unwrap();
let fast_field_reader: I64FastFieldReader =
fast_field_readers.open_reader(i64_field).unwrap();
I64FastFieldReader::open(fast_fields_composite.open_read(i64_field).unwrap());
assert_eq!(fast_field_reader.get(0u32), 0i64);
}
}
@@ -305,9 +312,10 @@ mod tests {
}
let source = directory.open_read(&path).unwrap();
{
let fast_field_readers = FastFieldsReader::from_source(source).unwrap();
let fast_fields_composite = CompositeFile::open(source).unwrap();
let fast_field_reader: U64FastFieldReader =
fast_field_readers.open_reader(*FIELD).unwrap();
U64FastFieldReader::open(fast_fields_composite.open_read(*FIELD).unwrap());
let mut a = 0u64;
for _ in 0..n {
assert_eq!(fast_field_reader.get(a as u32), permutation[a as usize]);
@@ -359,9 +367,11 @@ mod tests {
}
let source = directory.open_read(&path).unwrap();
{
let fast_field_readers = FastFieldsReader::from_source(source).unwrap();
let fast_fields_composite = CompositeFile::open(source).unwrap();
let fast_field_reader: U64FastFieldReader =
fast_field_readers.open_reader(*FIELD).unwrap();
U64FastFieldReader::open(fast_fields_composite.open_read(*FIELD).unwrap());
b.iter(|| {
let n = test::black_box(7000u32);
let mut a = 0u64;
@@ -390,9 +400,10 @@ mod tests {
}
let source = directory.open_read(&path).unwrap();
{
let fast_field_readers = FastFieldsReader::from_source(source).unwrap();
let fast_fields_composite = CompositeFile::open(source).unwrap();
let fast_field_reader: U64FastFieldReader =
fast_field_readers.open_reader(*FIELD).unwrap();
U64FastFieldReader::open(fast_fields_composite.open_read(*FIELD).unwrap());
b.iter(|| {
let n = test::black_box(1000u32);
let mut a = 0u32;

60
src/fastfield/reader.rs
View File

@@ -148,20 +148,24 @@ impl From<Vec<u64>> for U64FastFieldReader {
fast_field_writers.serialize(&mut serializer).unwrap();
serializer.close().unwrap();
}
panic!("TODO fix me");
/*
directory
.open_read(path)
.chain_err(|| "Failed to open the file")
.and_then(|source| {
FastFieldsReader::from_source(source)
.chain_err(|| "Failed to read the file.")
})
.and_then(|ff_readers| {
ff_readers
.open_reader(field)
.ok_or_else(|| "Failed to find the requested field".into())
})
.expect("This should never happen, please report.")
CompositeFile::open(source)
.chain_err(|| "Failed to read the file.")
})
.and_then(|composite_file| {
composite_file.open_read(field)
ff_readers
.open_reader(field)
.ok_or_else(|| "Failed to find the requested field".into())
})
.expect("This should never happen, please report.")
*/
}
}
@@ -231,41 +235,3 @@ impl FastFieldReader for I64FastFieldReader {
}
}
}
/// The `FastFieldsReader` is the datastructure containing
/// all of the fast fields' data.
///
/// It contains a mapping that associated these fields to
/// the proper slice in the fastfield reader file.
pub struct FastFieldsReader {
composite_file: CompositeFile,
}
impl FastFieldsReader {
/// Opens a `FastFieldsReader`
///
/// When opening the fast field reader, the
/// the list of the offset is read (as a footer of the
/// data file).
pub fn from_source(source: ReadOnlySource) -> io::Result<FastFieldsReader> {
Ok(FastFieldsReader {
composite_file: CompositeFile::open(source)?,
})
}
/// Returns the u64 fast value reader if the field
/// is a u64 field indexed as "fast".
///
/// Return None if the field is not a u64 field
/// indexed with the fast option.
///
/// # Panics
/// May panic if the index is corrupted.
pub fn open_reader<FFReader: FastFieldReader>(&self, field: Field) -> Option<FFReader> {
self.composite_file
.open_read(field)
.map(FFReader::open)
}
}

4
src/indexer/merger.rs
View File

@@ -58,7 +58,9 @@ fn extract_fieldnorm_reader(segment_reader: &SegmentReader,
fn extract_fast_field_reader(segment_reader: &SegmentReader,
field: Field)
-> Option<U64FastFieldReader> {
segment_reader.fast_fields_reader().open_reader(field)
segment_reader.get_fast_field_reader(field)
.ok()
}
struct DeltaComputer {