diff --git a/src/core/searcher.rs b/src/core/searcher.rs index d84ad22a3..eeac26072 100644 --- a/src/core/searcher.rs +++ b/src/core/searcher.rs @@ -23,7 +23,6 @@ pub struct Searcher { segment_readers: Vec, } - impl Searcher { /// Fetches a document from tantivy's store given a `DocAddress`. /// diff --git a/src/core/segment_reader.rs b/src/core/segment_reader.rs index 336496018..e3d203488 100644 --- a/src/core/segment_reader.rs +++ b/src/core/segment_reader.rs @@ -17,7 +17,7 @@ use common::CompositeFile; use std::fmt; use core::FieldReader; use schema::Field; -use fastfield::{FastFieldsReader, FastFieldReader, U64FastFieldReader}; +use fastfield::{FastFieldReader, U64FastFieldReader}; use schema::Schema; @@ -43,10 +43,10 @@ pub struct SegmentReader { termdict_composite: CompositeFile, postings_composite: CompositeFile, positions_composite: CompositeFile, + fast_fields_composite: CompositeFile, + fieldnorms_composite: CompositeFile, store_reader: StoreReader, - fast_fields_reader: Arc, - fieldnorms_reader: Arc, delete_bitset: DeleteBitSet, schema: Schema, } @@ -75,11 +75,6 @@ impl SegmentReader { self.delete_bitset.len() as DocId } - #[doc(hidden)] - pub fn fast_fields_reader(&self) -> &FastFieldsReader { - &*self.fast_fields_reader - } - /// Accessor to a segment's fast field reader given a field. /// /// Returns the u64 fast value reader if the field @@ -91,16 +86,17 @@ impl SegmentReader { /// # Panics /// May panic if the index is corrupted. pub fn get_fast_field_reader - (&self, - field: Field) - -> fastfield::Result { + (&self, field: Field) -> fastfield::Result { let field_entry = self.schema.get_field_entry(field); if !TFastFieldReader::is_enabled(field_entry.field_type()) { Err(FastFieldNotAvailableError::new(field_entry)) } else { - Ok(self.fast_fields_reader - .open_reader(field) - .expect("Fast field file corrupted.")) + self.fast_fields_composite + .open_read(field) + .ok_or_else(|| { + FastFieldNotAvailableError::new(field_entry) + }) + .map(TFastFieldReader::open) } } @@ -113,7 +109,9 @@ impl SegmentReader { /// They are simply stored as a fast field, serialized in /// the `.fieldnorm` file of the segment. pub fn get_fieldnorms_reader(&self, field: Field) -> Option { - self.fieldnorms_reader.open_reader(field) + self.fieldnorms_composite + .open_read(field) + .map(U64FastFieldReader::open) } /// Accessor to the segment's `StoreReader`. @@ -143,11 +141,11 @@ impl SegmentReader { }; - let fast_field_data = segment.open_read(SegmentComponent::FASTFIELDS)?; - let fast_fields_reader = FastFieldsReader::from_source(fast_field_data)?; + let fast_fields_data = segment.open_read(SegmentComponent::FASTFIELDS)?; + let fast_fields_composite = CompositeFile::open(fast_fields_data)?; let fieldnorms_data = segment.open_read(SegmentComponent::FIELDNORMS)?; - let fieldnorms_reader = FastFieldsReader::from_source(fieldnorms_data)?; + let fieldnorms_composite = CompositeFile::open(fieldnorms_data)?; let delete_bitset = if segment.meta().has_deletes() { @@ -161,12 +159,12 @@ impl SegmentReader { Ok(SegmentReader { field_reader_cache: Arc::new(RwLock::new(HashMap::new())), segment_meta: segment.meta().clone(), - postings_composite: postings_composite, termdict_composite: termdict_composite, + postings_composite: postings_composite, + fast_fields_composite: fast_fields_composite, + fieldnorms_composite: fieldnorms_composite, segment_id: segment.id(), store_reader: store_reader, - fast_fields_reader: Arc::new(fast_fields_reader), - fieldnorms_reader: Arc::new(fieldnorms_reader), delete_bitset: delete_bitset, positions_composite: positions_composite, schema: schema, diff --git a/src/fastfield/mod.rs b/src/fastfield/mod.rs index 31b241388..cff0f151e 100644 --- a/src/fastfield/mod.rs +++ b/src/fastfield/mod.rs @@ -32,7 +32,7 @@ mod delete; pub use self::delete::write_delete_bitset; pub use self::delete::DeleteBitSet; pub use self::writer::{FastFieldsWriter, IntFastFieldWriter}; -pub use self::reader::{FastFieldsReader, U64FastFieldReader, I64FastFieldReader}; +pub use self::reader::{U64FastFieldReader, I64FastFieldReader}; pub use self::reader::FastFieldReader; pub use self::serializer::FastFieldSerializer; pub use self::error::{Result, FastFieldNotAvailableError}; @@ -51,6 +51,7 @@ mod tests { use fastfield::FastFieldReader; use rand::Rng; use rand::SeedableRng; + use common::CompositeFile; use rand::XorShiftRng; lazy_static! { @@ -96,6 +97,8 @@ mod tests { { assert_eq!(source.len(), 35 as usize); } + // TODO uncomment + /* { let fast_field_readers = FastFieldsReader::from_source(source).unwrap(); let fast_field_reader: U64FastFieldReader = @@ -104,6 +107,7 @@ mod tests { assert_eq!(fast_field_reader.get(1), 14u64); assert_eq!(fast_field_reader.get(2), 2u64); } + */ } #[test] @@ -131,9 +135,9 @@ mod tests { assert_eq!(source.len(), 60 as usize); } { - let fast_field_readers = FastFieldsReader::from_source(source).unwrap(); + let fast_fields_composite = CompositeFile::open(source).unwrap(); let fast_field_reader: U64FastFieldReader = - fast_field_readers.open_reader(*FIELD).unwrap(); + U64FastFieldReader::open(fast_fields_composite.open_read(*FIELD).unwrap()); assert_eq!(fast_field_reader.get(0), 4u64); assert_eq!(fast_field_reader.get(1), 14_082_001u64); assert_eq!(fast_field_reader.get(2), 3_052u64); @@ -167,9 +171,9 @@ mod tests { assert_eq!(source.len(), 33 as usize); } { - let fast_field_readers = FastFieldsReader::from_source(source).unwrap(); + let fast_fields_composite = CompositeFile::open(source).unwrap(); let fast_field_reader: U64FastFieldReader = - fast_field_readers.open_reader(*FIELD).unwrap(); + U64FastFieldReader::open(fast_fields_composite.open_read(*FIELD).unwrap()); for doc in 0..10_000 { assert_eq!(fast_field_reader.get(doc), 100_000u64); } @@ -200,9 +204,10 @@ mod tests { assert_eq!(source.len(), 80041 as usize); } { - let fast_field_readers = FastFieldsReader::from_source(source).unwrap(); + let fast_fields_composite = CompositeFile::open(source).unwrap(); let fast_field_reader: U64FastFieldReader = - fast_field_readers.open_reader(*FIELD).unwrap(); + U64FastFieldReader::open(fast_fields_composite.open_read(*FIELD).unwrap()); + assert_eq!(fast_field_reader.get(0), 0u64); for doc in 1..10_001 { assert_eq!(fast_field_reader.get(doc), @@ -236,9 +241,10 @@ mod tests { assert_eq!(source.len(), 17708 as usize); } { - let fast_field_readers = FastFieldsReader::from_source(source).unwrap(); + let fast_fields_composite = CompositeFile::open(source).unwrap(); let fast_field_reader: I64FastFieldReader = - fast_field_readers.open_reader(i64_field).unwrap(); + I64FastFieldReader::open(fast_fields_composite.open_read(i64_field).unwrap()); + assert_eq!(fast_field_reader.min_value(), -100i64); assert_eq!(fast_field_reader.max_value(), 9_999i64); for (doc, i) in (-100i64..10_000i64).enumerate() { @@ -272,9 +278,10 @@ mod tests { let source = directory.open_read(&path).unwrap(); { - let fast_field_readers = FastFieldsReader::from_source(source).unwrap(); + + let fast_fields_composite = CompositeFile::open(source).unwrap(); let fast_field_reader: I64FastFieldReader = - fast_field_readers.open_reader(i64_field).unwrap(); + I64FastFieldReader::open(fast_fields_composite.open_read(i64_field).unwrap()); assert_eq!(fast_field_reader.get(0u32), 0i64); } } @@ -305,9 +312,10 @@ mod tests { } let source = directory.open_read(&path).unwrap(); { - let fast_field_readers = FastFieldsReader::from_source(source).unwrap(); + let fast_fields_composite = CompositeFile::open(source).unwrap(); let fast_field_reader: U64FastFieldReader = - fast_field_readers.open_reader(*FIELD).unwrap(); + U64FastFieldReader::open(fast_fields_composite.open_read(*FIELD).unwrap()); + let mut a = 0u64; for _ in 0..n { assert_eq!(fast_field_reader.get(a as u32), permutation[a as usize]); @@ -359,9 +367,11 @@ mod tests { } let source = directory.open_read(&path).unwrap(); { - let fast_field_readers = FastFieldsReader::from_source(source).unwrap(); + let fast_fields_composite = CompositeFile::open(source).unwrap(); let fast_field_reader: U64FastFieldReader = - fast_field_readers.open_reader(*FIELD).unwrap(); + U64FastFieldReader::open(fast_fields_composite.open_read(*FIELD).unwrap()); + + b.iter(|| { let n = test::black_box(7000u32); let mut a = 0u64; @@ -390,9 +400,10 @@ mod tests { } let source = directory.open_read(&path).unwrap(); { - let fast_field_readers = FastFieldsReader::from_source(source).unwrap(); + let fast_fields_composite = CompositeFile::open(source).unwrap(); let fast_field_reader: U64FastFieldReader = - fast_field_readers.open_reader(*FIELD).unwrap(); + U64FastFieldReader::open(fast_fields_composite.open_read(*FIELD).unwrap()); + b.iter(|| { let n = test::black_box(1000u32); let mut a = 0u32; diff --git a/src/fastfield/reader.rs b/src/fastfield/reader.rs index 2ec8f66fc..45848293d 100644 --- a/src/fastfield/reader.rs +++ b/src/fastfield/reader.rs @@ -148,20 +148,24 @@ impl From> for U64FastFieldReader { fast_field_writers.serialize(&mut serializer).unwrap(); serializer.close().unwrap(); } + panic!("TODO fix me"); + /* directory .open_read(path) .chain_err(|| "Failed to open the file") .and_then(|source| { - FastFieldsReader::from_source(source) - .chain_err(|| "Failed to read the file.") - }) - .and_then(|ff_readers| { - ff_readers - .open_reader(field) - .ok_or_else(|| "Failed to find the requested field".into()) - }) - .expect("This should never happen, please report.") + CompositeFile::open(source) + .chain_err(|| "Failed to read the file.") + }) + .and_then(|composite_file| { + composite_file.open_read(field) + ff_readers + .open_reader(field) + .ok_or_else(|| "Failed to find the requested field".into()) + }) + .expect("This should never happen, please report.") + */ } } @@ -231,41 +235,3 @@ impl FastFieldReader for I64FastFieldReader { } } } - - - -/// The `FastFieldsReader` is the datastructure containing -/// all of the fast fields' data. -/// -/// It contains a mapping that associated these fields to -/// the proper slice in the fastfield reader file. -pub struct FastFieldsReader { - composite_file: CompositeFile, -} - -impl FastFieldsReader { - /// Opens a `FastFieldsReader` - /// - /// When opening the fast field reader, the - /// the list of the offset is read (as a footer of the - /// data file). - pub fn from_source(source: ReadOnlySource) -> io::Result { - Ok(FastFieldsReader { - composite_file: CompositeFile::open(source)?, - }) - } - - /// Returns the u64 fast value reader if the field - /// is a u64 field indexed as "fast". - /// - /// Return None if the field is not a u64 field - /// indexed with the fast option. - /// - /// # Panics - /// May panic if the index is corrupted. - pub fn open_reader(&self, field: Field) -> Option { - self.composite_file - .open_read(field) - .map(FFReader::open) - } -} diff --git a/src/indexer/merger.rs b/src/indexer/merger.rs index 48aa695aa..2d2385cb9 100644 --- a/src/indexer/merger.rs +++ b/src/indexer/merger.rs @@ -58,7 +58,9 @@ fn extract_fieldnorm_reader(segment_reader: &SegmentReader, fn extract_fast_field_reader(segment_reader: &SegmentReader, field: Field) -> Option { - segment_reader.fast_fields_reader().open_reader(field) + segment_reader.get_fast_field_reader(field) + .ok() + } struct DeltaComputer {