diff --git a/src/core/merger.rs b/src/core/merger.rs index 91bf9dc7d..e67b2153b 100644 --- a/src/core/merger.rs +++ b/src/core/merger.rs @@ -152,30 +152,29 @@ impl IndexMerger { fn write_fast_fields(&self, fast_field_serializer: &mut FastFieldSerializer) -> io::Result<()> { // TODO implement fast field - // for field in self.schema - // .get_u32_fields() - // .iter() - // .enumerate() - // .filter(|&(_, field_entry)| field_entry.option.is_fast()) - // .map(|(field_id, _)| Field(field_id as u8)) { - // let mut u32_readers = Vec::new(); - // let mut min_val = u32::min_value(); - // let mut max_val = 0; - // for reader in self.readers.iter() { - // let u32_reader = try!(reader.get_fast_field_reader(&field)); - // min_val = min(min_val, u32_reader.min_val()); - // max_val = max(max_val, u32_reader.max_val()); - // u32_readers.push((reader.max_doc(), u32_reader)); - // } - // try!(fast_field_serializer.new_u32_fast_field(field, min_val, max_val)); - // for (max_doc, u32_reader) in u32_readers { - // for doc_id in 0..max_doc { - // let val = u32_reader.get(doc_id); - // try!(fast_field_serializer.add_val(val)); - // } - // } - // try!(fast_field_serializer.close_field()); - // } + for field in self.schema.fields() + .iter() + .enumerate() + .filter(|&(_, field_entry)| field_entry.is_u32_fast()) + .map(|(field_id, _)| Field(field_id as u8)) { + let mut u32_readers = Vec::new(); + let mut min_val = u32::min_value(); + let mut max_val = 0; + for reader in self.readers.iter() { + let u32_reader = try!(reader.get_fast_field_reader(field)); + min_val = min(min_val, u32_reader.min_val()); + max_val = max(max_val, u32_reader.max_val()); + u32_readers.push((reader.max_doc(), u32_reader)); + } + try!(fast_field_serializer.new_u32_fast_field(field, min_val, max_val)); + for (max_doc, u32_reader) in u32_readers { + for doc_id in 0..max_doc { + let val = u32_reader.get(doc_id); + try!(fast_field_serializer.add_val(val)); + } + } + try!(fast_field_serializer.close_field()); + } Ok(()) } diff --git a/src/core/reader.rs b/src/core/reader.rs index e97632a47..5e0795bf3 100644 --- a/src/core/reader.rs +++ b/src/core/reader.rs @@ -98,8 +98,8 @@ impl SegmentReader { FieldEntry::Text(_, _) => { Err(io::Error::new(io::ErrorKind::Other, "fast field are not yet supported for text fields.")) }, - FieldEntry::U32(_, ref options) => { - //if options. + FieldEntry::U32(_, _) => { + // TODO check that the schema allows that //Err(io::Error::new(io::ErrorKind::Other, "fast field are not yet supported for text fields.")) self.fast_fields_reader.get_field(field) }, diff --git a/src/core/writer.rs b/src/core/writer.rs index 4db6e9966..f65e35745 100644 --- a/src/core/writer.rs +++ b/src/core/writer.rs @@ -23,6 +23,7 @@ use std::sync::mpsc::Receiver; use std::thread::JoinHandle; use std::sync::Arc; use core::merger::IndexMerger; +use schema::FieldValue; pub struct IndexWriter { threads: Vec>, @@ -189,14 +190,13 @@ impl SegmentWriter { } } self.fast_field_writers.add_document(&doc); - - // TODO add stored field - // let stored_fieldvalues: Vec<&TextFieldValue> = doc - // .text_fields() - // .filter(|text_field_value| schema.text_field_options(&text_field_value.field).is_stored()) - // .collect(); - // let doc_writer = self.segment_serializer.get_store_writer(); - // try!(doc_writer.store(&stored_fieldvalues)); + let stored_fieldvalues: Vec<&FieldValue> = doc + .get_fields() + .iter() + .filter(|field_value| schema.field_entry(field_value.field()).is_stored()) + .collect(); + let doc_writer = self.segment_serializer.get_store_writer(); + try!(doc_writer.store(&stored_fieldvalues)); self.max_doc += 1; Ok(()) } diff --git a/src/fastfield/reader.rs b/src/fastfield/reader.rs index a8d44c1a9..0765e1fca 100644 --- a/src/fastfield/reader.rs +++ b/src/fastfield/reader.rs @@ -6,7 +6,6 @@ use std::ops::Deref; use directory::ReadOnlySource; use fastfield::DividerU32; use common::BinarySerializable; -use schema::FieldEntry; use DocId; use schema::Field; diff --git a/src/fastfield/writer.rs b/src/fastfield/writer.rs index c0d3e05e3..e0e08ada7 100644 --- a/src/fastfield/writer.rs +++ b/src/fastfield/writer.rs @@ -1,5 +1,4 @@ use schema::{Schema, FieldValue, Field, Document}; -use schema::FieldEntry; use fastfield::FastFieldSerializer; use std::io; @@ -7,18 +6,6 @@ pub struct U32FastFieldsWriter { field_writers: Vec, } - -fn is_u32_fast(field_entry: &FieldEntry) -> bool { - match field_entry { - &FieldEntry::U32(_, ref options) => { - options.is_fast() - } - _ => { - false - } - } -} - impl U32FastFieldsWriter { pub fn from_schema(schema: &Schema) -> U32FastFieldsWriter { @@ -26,7 +13,7 @@ impl U32FastFieldsWriter { let u32_fields: Vec = schema.fields() .iter() .enumerate() - .filter(|&(_, field_entry)| is_u32_fast(field_entry)) + .filter(|&(_, field_entry)| field_entry.is_u32_fast()) .map(|(field_id, _)| Field(field_id as u8)) .collect(); U32FastFieldsWriter::new(u32_fields) diff --git a/src/schema/document.rs b/src/schema/document.rs index d7069e6ca..c70e43ef2 100644 --- a/src/schema/document.rs +++ b/src/schema/document.rs @@ -1,24 +1,12 @@ -use std::slice; - use super::*; + /// /// Document are really just a list of field values. /// -/// # Examples -/// -/// ``` -/// use tantivy::schema::Schema; -/// use tantivy::schema::TEXT; -/// -/// let mut schema = Schema::new(); -/// schema.add_text_field("body", &TEXT); -/// let field_text = schema.text_field("body"); -/// ``` -/// #[derive(Debug)] pub struct Document { - pub field_values: Vec, + field_values: Vec, } impl Document { @@ -28,13 +16,7 @@ impl Document { field_values: Vec::new(), } } - - pub fn from(field_values: Vec) -> Document { - Document { - field_values: field_values, - } - } - + pub fn len(&self,) -> usize { self.field_values.len() } @@ -50,33 +32,7 @@ impl Document { pub fn add(&mut self, field_value: FieldValue) { self.field_values.push(field_value); } - - // pub fn text_fields<'a>(&'a self,) -> slice::Iter<'a, TextFieldValue> { - // self.text_field_values.iter() - // } - - // pub fn u32_fields<'a>(&'a self,) -> slice::Iter<'a, U32FieldValue> { - // self.u32_field_values.iter() - // } - - // pub fn get_u32(&self, field: &U32Field) -> Option { - // self.u32_field_values - // .iter() - // .filter(|field_value| field_value.field == *field) - // .map(|field_value| &field_value.value) - // .cloned() - // .next() - // } - - // pub fn get_texts<'a>(&'a self, field: &TextField) -> Vec<&'a String> { - // self.text_field_values - // .iter() - // .filter(|field_value| field_value.field == *field) - // .map(|field_value| &field_value.text) - // .collect() - // } - - + pub fn get_fields(&self) -> &Vec { &self.field_values } @@ -95,3 +51,11 @@ impl Document { .next() } } + +impl From> for Document { + fn from(field_values: Vec) -> Document { + Document { + field_values: field_values + } + } +} diff --git a/src/schema/field.rs b/src/schema/field.rs index a5338b4c7..d67b30bda 100644 --- a/src/schema/field.rs +++ b/src/schema/field.rs @@ -1,13 +1,7 @@ -use std::io::Write; use std::io; - +use std::io::Write; use std::io::Read; use common::BinarySerializable; -use rustc_serialize::Decoder; -use rustc_serialize::Encoder; -use std::ops::BitOr; -use std::marker::Copy; - // TODO impl Copy trait diff --git a/src/schema/field_entry.rs b/src/schema/field_entry.rs index dd743dd71..2253c4113 100644 --- a/src/schema/field_entry.rs +++ b/src/schema/field_entry.rs @@ -1,8 +1,6 @@ -use rustc_serialize::Decodable; use rustc_serialize::Encodable; use rustc_serialize::Decoder; use rustc_serialize::Encoder; -use schema::Field; use schema::TextOptions; use schema::U32Options; @@ -22,5 +20,28 @@ impl FieldEntry { field_name } } - } + } + + pub fn is_u32_fast(&self,) -> bool { + match self { + &FieldEntry::U32(_, ref options) => { + options.is_fast() + } + _ => { + false + } + } + } + + pub fn is_stored(&self,) -> bool { + match self { + &FieldEntry::U32(_, ref options) => { + // TODO handle stored u32 + options.is_stored() + } + &FieldEntry::Text(_, ref options) => { + options.is_stored() + } + } + } } \ No newline at end of file diff --git a/src/schema/field_value.rs b/src/schema/field_value.rs index 522874776..2be318f67 100644 --- a/src/schema/field_value.rs +++ b/src/schema/field_value.rs @@ -66,8 +66,8 @@ impl BinarySerializable for FieldValue { Ok(written_size) } fn deserialize(reader: &mut Read) -> io::Result { - let TYPE_CODE = try!(u8::deserialize(reader)); - match TYPE_CODE { + let type_code = try!(u8::deserialize(reader)); + match type_code { TEXT_CODE => { let field = try!(Field::deserialize(reader)); let text = try!(String::deserialize(reader)); @@ -79,7 +79,7 @@ impl BinarySerializable for FieldValue { Ok(FieldValue::U32(field, value)) } _ => { - Err(io::Error::new(io::ErrorKind::InvalidData, format!("No field type is associated with code {:?}", TYPE_CODE))) + Err(io::Error::new(io::ErrorKind::InvalidData, format!("No field type is associated with code {:?}", type_code))) } } } diff --git a/src/schema/schema.rs b/src/schema/schema.rs index af87e37aa..90ba3b594 100644 --- a/src/schema/schema.rs +++ b/src/schema/schema.rs @@ -4,23 +4,8 @@ use rustc_serialize::Decodable; use rustc_serialize::Encodable; use rustc_serialize::Decoder; use rustc_serialize::Encoder; -use std::borrow::Borrow; use super::*; -// #[derive(Clone, Debug, RustcDecodable, RustcEncodable)] -// pub struct TextFieldEntry { -// name: String, -// option: TextOptions, -// } - - -// #[derive(Clone, Debug, RustcDecodable, RustcEncodable)] -// pub struct U32FieldEntry { -// pub name: String, -// pub option: U32Options, -// } - - /// Tantivy has a very strict schema. /// You need to specify in advance, whether a field is indexed or not, @@ -38,13 +23,13 @@ use super::*; /// fn create_schema() -> Schema { /// let mut schema = Schema::new(); /// let str_fieldtype = TextOptions::new(); -/// let id_field = schema.add_text_field("id", &STRING); -/// let url_field = schema.add_text_field("url", &STRING); -/// let body_field = schema.add_text_field("body", &TEXT); -/// let id_field = schema.add_text_field("id", &STRING); -/// let url_field = schema.add_text_field("url", &STRING); -/// let title_field = schema.add_text_field("title", &TEXT); -/// let body_field = schema.add_text_field("body", &TEXT); +/// let id_field = schema.add_text_field("id", STRING); +/// let url_field = schema.add_text_field("url", STRING); +/// let body_field = schema.add_text_field("body", TEXT); +/// let id_field = schema.add_text_field("id", STRING); +/// let url_field = schema.add_text_field("url", STRING); +/// let title_field = schema.add_text_field("title", TEXT); +/// let body_field = schema.add_text_field("body", TEXT); /// schema /// } /// @@ -102,31 +87,6 @@ impl Schema { &self.fields } - // pub fn get_u32_fields(&self,) -> &Vec { - // &self.u32_fields - // } - - /// Given a name, returns the field handle, as well as its associated TextOptions - // pub fn get_text_field(&self, field_name: &str) -> Option<(TextField, TextOptions)> { - // self.text_fields_map - // .get(field_name) - // .map(|&TextField(field_id)| { - // let field_options = self.text_fields[field_id as usize].option.clone(); - // (TextField(field_id), field_options) - // }) - // } - - // pub fn get_u32_field(&self, field_name: &str) -> Option<(U32Field, U32Options)> { - // self.u32_fields_map - // .get(field_name) - // .map(|&U32Field(field_id)| { - // let u32_field_options = self.u32_fields[field_id as usize].option.clone(); - // (U32Field(field_id), u32_field_options) - // }) - // } - - - /// Returns the field options associated with a given name. /// /// # Panics @@ -140,36 +100,6 @@ impl Schema { self.fields_map.get(field_name).map(|field| field.clone()) } - // pub fn u32_field(&self, fieldname: &str) -> U32Field { - // self.u32_fields_map.get(fieldname).map(|field| field.clone()).unwrap() - // } - - /// Returns the field options associated to a field handle. - // pub fn text_field_options(&self, field: &TextField) -> TextOptions { - // let TextField(field_id) = *field; - // self.text_fields[field_id as usize].option.clone() - // } - - - // pub fn u32_field_options(&self, field: &U32Field) -> U32Options { - // let U32Field(field_id) = *field; - // self.u32_fields[field_id as usize].option.clone() - // } - - // /// Creates a new field. - // /// Return the associated field handle. - // pub fn add_text_field>(&mut self, field_name_str: &str, field_options: RefTextOptions) -> TextField { - // let field = TextField(self.text_fields.len() as u8); - // // TODO case if field already exists - // let field_name = String::from(field_name_str); - // self.text_fields.push(TextFieldEntry { - // name: field_name.clone(), - // option: field_options.borrow().clone(), - // }); - // self.text_fields_map.insert(field_name, field.clone()); - // field - // } - /// Creates a new field. /// Return the associated field handle. pub fn add_u32_field( diff --git a/src/schema/text_field.rs b/src/schema/text_field.rs index 9af8326d3..b331f77e2 100644 --- a/src/schema/text_field.rs +++ b/src/schema/text_field.rs @@ -1,8 +1,3 @@ -use std::io::Write; -use std::io; - -use std::io::Read; -use common::BinarySerializable; use rustc_serialize::Decoder; use rustc_serialize::Encoder; use std::ops::BitOr; diff --git a/src/schema/u32_field.rs b/src/schema/u32_field.rs index 11164bc24..21e145a6f 100644 --- a/src/schema/u32_field.rs +++ b/src/schema/u32_field.rs @@ -1,32 +1,6 @@ -use std::io; -use std::io::Write; -use std::io::Read; - -use common::BinarySerializable; use rustc_serialize::Decoder; use rustc_serialize::Encoder; -#[derive(Clone,Debug,PartialEq,PartialOrd,Eq,Hash)] -pub struct U32Field(pub u8); - -impl BinarySerializable for U32Field { - fn serialize(&self, writer: &mut Write) -> io::Result { - let U32Field(field_id) = *self; - field_id.serialize(writer) - } - - fn deserialize(reader: &mut Read) -> io::Result { - u8::deserialize(reader).map(U32Field) - } -} - -#[derive(Clone,Debug,PartialEq,PartialOrd,Eq)] -pub struct U32FieldValue { - pub field: U32Field, - pub value: u32, -} - - #[derive(Clone,Debug,PartialEq,Eq, RustcDecodable, RustcEncodable)] pub struct U32Options { indexed: bool, @@ -34,7 +8,6 @@ pub struct U32Options { stored: bool, } - impl U32Options { pub fn new() -> U32Options { @@ -44,7 +17,11 @@ impl U32Options { stored: false, } } - + + pub fn is_stored(&self,) -> bool { + self.stored + } + pub fn is_indexed(&self,) -> bool { self.indexed } diff --git a/src/store/reader.rs b/src/store/reader.rs index 5e5c5412f..99c29ba76 100644 --- a/src/store/reader.rs +++ b/src/store/reader.rs @@ -78,9 +78,7 @@ impl StoreReader { let field_value = try!(FieldValue::deserialize(&mut cursor)); field_values.push(field_value); } - Ok(Document { - field_values: Vec::new(), - }) + Ok(Document::from(field_values)) } pub fn new(data: ReadOnlySource) -> StoreReader {