rust/tantivy
1
0
Fork 0
mirror of https://github.com/quickwit-oss/tantivy.git synced 2026-10-29 14:57:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Test passing

Browse Source
This commit is contained in:
Paul Masurel
2016-06-19 16:08:55 +09:00
parent 19b3835b53
commit 9d8b3759d8
13 changed files with 87 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/merger.rs
+23 -24
View File
@@ -152,30 +152,29 @@ impl IndexMerger {
fn write_fast_fields(&self, fast_field_serializer: &mut FastFieldSerializer) -> io::Result<()> {
// TODO implement fast field
// for field in self.schema
// .get_u32_fields()
// .iter()
// .enumerate()
// .filter(|&(_, field_entry)| field_entry.option.is_fast())
// .map(|(field_id, _)| Field(field_id as u8)) {
// let mut u32_readers = Vec::new();
// let mut min_val = u32::min_value();
// let mut max_val = 0;
// for reader in self.readers.iter() {
// let u32_reader = try!(reader.get_fast_field_reader(&field));
// min_val = min(min_val, u32_reader.min_val());
// max_val = max(max_val, u32_reader.max_val());
// u32_readers.push((reader.max_doc(), u32_reader));
// }
// try!(fast_field_serializer.new_u32_fast_field(field, min_val, max_val));
// for (max_doc, u32_reader) in u32_readers {
// for doc_id in 0..max_doc {
// let val = u32_reader.get(doc_id);
// try!(fast_field_serializer.add_val(val));
// }
// }
// try!(fast_field_serializer.close_field());
// }
for field in self.schema.fields()
.iter()
.enumerate()
.filter(|&(_, field_entry)| field_entry.is_u32_fast())
.map(|(field_id, _)| Field(field_id as u8)) {
let mut u32_readers = Vec::new();
let mut min_val = u32::min_value();
let mut max_val = 0;
for reader in self.readers.iter() {
let u32_reader = try!(reader.get_fast_field_reader(field));
min_val = min(min_val, u32_reader.min_val());
max_val = max(max_val, u32_reader.max_val());
u32_readers.push((reader.max_doc(), u32_reader));
}
try!(fast_field_serializer.new_u32_fast_field(field, min_val, max_val));
for (max_doc, u32_reader) in u32_readers {
for doc_id in 0..max_doc {
let val = u32_reader.get(doc_id);
try!(fast_field_serializer.add_val(val));
}
}
try!(fast_field_serializer.close_field());
}
Ok(())
}
src/core/reader.rs
+2 -2
View File
@@ -98,8 +98,8 @@ impl SegmentReader {
FieldEntry::Text(_, _) => {
Err(io::Error::new(io::ErrorKind::Other, "fast field are not yet supported for text fields."))
},
FieldEntry::U32(_, ref options) => {
//if options.
FieldEntry::U32(_, _) => {
// TODO check that the schema allows that
//Err(io::Error::new(io::ErrorKind::Other, "fast field are not yet supported for text fields."))
self.fast_fields_reader.get_field(field)
},
src/core/writer.rs
+8 -8
View File
@@ -23,6 +23,7 @@ use std::sync::mpsc::Receiver;
use std::thread::JoinHandle;
use std::sync::Arc;
use core::merger::IndexMerger;
use schema::FieldValue;
pub struct IndexWriter {
threads: Vec<JoinHandle<()>>,
@@ -189,14 +190,13 @@ impl SegmentWriter {
}
}
self.fast_field_writers.add_document(&doc);
// TODO add stored field
// let stored_fieldvalues: Vec<&TextFieldValue> = doc
// .text_fields()
// .filter(|text_field_value| schema.text_field_options(&text_field_value.field).is_stored())
// .collect();
// let doc_writer = self.segment_serializer.get_store_writer();
// try!(doc_writer.store(&stored_fieldvalues));
let stored_fieldvalues: Vec<&FieldValue> = doc
.get_fields()
.iter()
.filter(|field_value| schema.field_entry(field_value.field()).is_stored())
.collect();
let doc_writer = self.segment_serializer.get_store_writer();
try!(doc_writer.store(&stored_fieldvalues));
self.max_doc += 1;
Ok(())
}
src/fastfield/reader.rs
-1
View File
@@ -6,7 +6,6 @@ use std::ops::Deref;
use directory::ReadOnlySource;
use fastfield::DividerU32;
use common::BinarySerializable;
use schema::FieldEntry;
use DocId;
use schema::Field;
src/fastfield/writer.rs
+1 -14
View File
@@ -1,5 +1,4 @@
use schema::{Schema, FieldValue, Field, Document};
use schema::FieldEntry;
use fastfield::FastFieldSerializer;
use std::io;
@@ -7,18 +6,6 @@ pub struct U32FastFieldsWriter {
field_writers: Vec<U32FastFieldWriter>,
}
fn is_u32_fast(field_entry: &FieldEntry) -> bool {
match field_entry {
&FieldEntry::U32(_, ref options) => {
options.is_fast()
}
_ => {
false
}
}
}
impl U32FastFieldsWriter {
pub fn from_schema(schema: &Schema) -> U32FastFieldsWriter {
@@ -26,7 +13,7 @@ impl U32FastFieldsWriter {
let u32_fields: Vec<Field> = schema.fields()
.iter()
.enumerate()
.filter(|&(_, field_entry)| is_u32_fast(field_entry))
.filter(|&(_, field_entry)| field_entry.is_u32_fast())
.map(|(field_id, _)| Field(field_id as u8))
.collect();
U32FastFieldsWriter::new(u32_fields)
src/schema/document.rs
+12 -48
View File
@@ -1,24 +1,12 @@
use std::slice;
use super::*;
///
/// Document are really just a list of field values.
///
/// # Examples
///
/// ```
/// use tantivy::schema::Schema;
/// use tantivy::schema::TEXT;
///
/// let mut schema = Schema::new();
/// schema.add_text_field("body", &TEXT);
/// let field_text = schema.text_field("body");
/// ```
///
#[derive(Debug)]
pub struct Document {
pub field_values: Vec<FieldValue>,
field_values: Vec<FieldValue>,
}
impl Document {
@@ -28,13 +16,7 @@ impl Document {
field_values: Vec::new(),
}
}
pub fn from(field_values: Vec<FieldValue>) -> Document {
Document {
field_values: field_values,
}
}
pub fn len(&self,) -> usize {
self.field_values.len()
}
@@ -50,33 +32,7 @@ impl Document {
pub fn add(&mut self, field_value: FieldValue) {
self.field_values.push(field_value);
}
// pub fn text_fields<'a>(&'a self,) -> slice::Iter<'a, TextFieldValue> {
// self.text_field_values.iter()
// }
// pub fn u32_fields<'a>(&'a self,) -> slice::Iter<'a, U32FieldValue> {
// self.u32_field_values.iter()
// }
// pub fn get_u32(&self, field: &U32Field) -> Option<u32> {
// self.u32_field_values
// .iter()
// .filter(|field_value| field_value.field == *field)
// .map(|field_value| &field_value.value)
// .cloned()
// .next()
// }
// pub fn get_texts<'a>(&'a self, field: &TextField) -> Vec<&'a String> {
// self.text_field_values
// .iter()
// .filter(|field_value| field_value.field == *field)
// .map(|field_value| &field_value.text)
// .collect()
// }
pub fn get_fields(&self) -> &Vec<FieldValue> {
&self.field_values
}
@@ -95,3 +51,11 @@ impl Document {
.next()
}
}
impl From<Vec<FieldValue>> for Document {
fn from(field_values: Vec<FieldValue>) -> Document {
Document {
field_values: field_values
}
}
}
src/schema/field.rs
+1 -7
View File
@@ -1,13 +1,7 @@
use std::io::Write;
use std::io;
use std::io::Write;
use std::io::Read;
use common::BinarySerializable;
use rustc_serialize::Decoder;
use rustc_serialize::Encoder;
use std::ops::BitOr;
use std::marker::Copy;
// TODO impl Copy trait
src/schema/field_entry.rs
+24 -3
View File
@@ -1,8 +1,6 @@
use rustc_serialize::Decodable;
use rustc_serialize::Encodable;
use rustc_serialize::Decoder;
use rustc_serialize::Encoder;
use schema::Field;
use schema::TextOptions;
use schema::U32Options;
@@ -22,5 +20,28 @@ impl FieldEntry {
field_name
}
}
}
}
pub fn is_u32_fast(&self,) -> bool {
match self {
&FieldEntry::U32(_, ref options) => {
options.is_fast()
}
_ => {
false
}
}
}
pub fn is_stored(&self,) -> bool {
match self {
&FieldEntry::U32(_, ref options) => {
// TODO handle stored u32
options.is_stored()
}
&FieldEntry::Text(_, ref options) => {
options.is_stored()
}
}
}
}
src/schema/field_value.rs
+3 -3
View File
@@ -66,8 +66,8 @@ impl BinarySerializable for FieldValue {
Ok(written_size)
}
fn deserialize(reader: &mut Read) -> io::Result<Self> {
let TYPE_CODE = try!(u8::deserialize(reader));
match TYPE_CODE {
let type_code = try!(u8::deserialize(reader));
match type_code {
TEXT_CODE => {
let field = try!(Field::deserialize(reader));
let text = try!(String::deserialize(reader));
@@ -79,7 +79,7 @@ impl BinarySerializable for FieldValue {
Ok(FieldValue::U32(field, value))
}
_ => {
Err(io::Error::new(io::ErrorKind::InvalidData, format!("No field type is associated with code {:?}", TYPE_CODE)))
Err(io::Error::new(io::ErrorKind::InvalidData, format!("No field type is associated with code {:?}", type_code)))
}
}
}
src/schema/schema.rs
+7 -77
View File
@@ -4,23 +4,8 @@ use rustc_serialize::Decodable;
use rustc_serialize::Encodable;
use rustc_serialize::Decoder;
use rustc_serialize::Encoder;
use std::borrow::Borrow;
use super::*;
// #[derive(Clone, Debug, RustcDecodable, RustcEncodable)]
// pub struct TextFieldEntry {
// name: String,
// option: TextOptions,
// }
// #[derive(Clone, Debug, RustcDecodable, RustcEncodable)]
// pub struct U32FieldEntry {
// pub name: String,
// pub option: U32Options,
// }
/// Tantivy has a very strict schema.
/// You need to specify in advance, whether a field is indexed or not,
@@ -38,13 +23,13 @@ use super::*;
/// fn create_schema() -> Schema {
/// let mut schema = Schema::new();
/// let str_fieldtype = TextOptions::new();
/// let id_field = schema.add_text_field("id", &STRING);
/// let url_field = schema.add_text_field("url", &STRING);
/// let body_field = schema.add_text_field("body", &TEXT);
/// let id_field = schema.add_text_field("id", &STRING);
/// let url_field = schema.add_text_field("url", &STRING);
/// let title_field = schema.add_text_field("title", &TEXT);
/// let body_field = schema.add_text_field("body", &TEXT);
/// let id_field = schema.add_text_field("id", STRING);
/// let url_field = schema.add_text_field("url", STRING);
/// let body_field = schema.add_text_field("body", TEXT);
/// let id_field = schema.add_text_field("id", STRING);
/// let url_field = schema.add_text_field("url", STRING);
/// let title_field = schema.add_text_field("title", TEXT);
/// let body_field = schema.add_text_field("body", TEXT);
/// schema
/// }
///
@@ -102,31 +87,6 @@ impl Schema {
&self.fields
}
// pub fn get_u32_fields(&self,) -> &Vec<U32FieldEntry> {
// &self.u32_fields
// }
/// Given a name, returns the field handle, as well as its associated TextOptions
// pub fn get_text_field(&self, field_name: &str) -> Option<(TextField, TextOptions)> {
// self.text_fields_map
// .get(field_name)
// .map(|&TextField(field_id)| {
// let field_options = self.text_fields[field_id as usize].option.clone();
// (TextField(field_id), field_options)
// })
// }
// pub fn get_u32_field(&self, field_name: &str) -> Option<(U32Field, U32Options)> {
// self.u32_fields_map
// .get(field_name)
// .map(|&U32Field(field_id)| {
// let u32_field_options = self.u32_fields[field_id as usize].option.clone();
// (U32Field(field_id), u32_field_options)
// })
// }
/// Returns the field options associated with a given name.
///
/// # Panics
@@ -140,36 +100,6 @@ impl Schema {
self.fields_map.get(field_name).map(|field| field.clone())
}
// pub fn u32_field(&self, fieldname: &str) -> U32Field {
// self.u32_fields_map.get(fieldname).map(|field| field.clone()).unwrap()
// }
/// Returns the field options associated to a field handle.
// pub fn text_field_options(&self, field: &TextField) -> TextOptions {
// let TextField(field_id) = *field;
// self.text_fields[field_id as usize].option.clone()
// }
// pub fn u32_field_options(&self, field: &U32Field) -> U32Options {
// let U32Field(field_id) = *field;
// self.u32_fields[field_id as usize].option.clone()
// }
// /// Creates a new field.
// /// Return the associated field handle.
// pub fn add_text_field<RefTextOptions: Borrow<TextOptions>>(&mut self, field_name_str: &str, field_options: RefTextOptions) -> TextField {
// let field = TextField(self.text_fields.len() as u8);
// // TODO case if field already exists
// let field_name = String::from(field_name_str);
// self.text_fields.push(TextFieldEntry {
// name: field_name.clone(),
// option: field_options.borrow().clone(),
// });
// self.text_fields_map.insert(field_name, field.clone());
// field
// }
/// Creates a new field.
/// Return the associated field handle.
pub fn add_u32_field(
src/schema/text_field.rs
-5
View File
@@ -1,8 +1,3 @@
use std::io::Write;
use std::io;
use std::io::Read;
use common::BinarySerializable;
use rustc_serialize::Decoder;
use rustc_serialize::Encoder;
use std::ops::BitOr;
src/schema/u32_field.rs
+5 -28
View File
@@ -1,32 +1,6 @@
use std::io;
use std::io::Write;
use std::io::Read;
use common::BinarySerializable;
use rustc_serialize::Decoder;
use rustc_serialize::Encoder;
#[derive(Clone,Debug,PartialEq,PartialOrd,Eq,Hash)]
pub struct U32Field(pub u8);
impl BinarySerializable for U32Field {
fn serialize(&self, writer: &mut Write) -> io::Result<usize> {
let U32Field(field_id) = *self;
field_id.serialize(writer)
}
fn deserialize(reader: &mut Read) -> io::Result<U32Field> {
u8::deserialize(reader).map(U32Field)
}
}
#[derive(Clone,Debug,PartialEq,PartialOrd,Eq)]
pub struct U32FieldValue {
pub field: U32Field,
pub value: u32,
}
#[derive(Clone,Debug,PartialEq,Eq, RustcDecodable, RustcEncodable)]
pub struct U32Options {
indexed: bool,
@@ -34,7 +8,6 @@ pub struct U32Options {
stored: bool,
}
impl U32Options {
pub fn new() -> U32Options {
@@ -44,7 +17,11 @@ impl U32Options {
stored: false,
}
}
pub fn is_stored(&self,) -> bool {
self.stored
}
pub fn is_indexed(&self,) -> bool {
self.indexed
}
src/store/reader.rs
+1 -3
View File
@@ -78,9 +78,7 @@ impl StoreReader {
let field_value = try!(FieldValue::deserialize(&mut cursor));
field_values.push(field_value);
}
Ok(Document {
field_values: Vec::new(),
})
Ok(Document::from(field_values))
}
pub fn new(data: ReadOnlySource) -> StoreReader {