rust/tantivy
1
0
Fork 0
mirror of https://github.com/quickwit-oss/tantivy.git synced 2026-05-21 18:50:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added test for field entry json ser/deser

Browse Source
This commit is contained in:
Paul Masurel
2016-06-19 22:47:43 +09:00
parent 9d8b3759d8
commit b93fa39910
10 changed files with 89 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/core/merger.rs
View File

@@ -1,5 +1,5 @@
use std::io;
use core::reader::SegmentReader;
use core::SegmentReader;
use core::index::Segment;
use DocId;
use core::index::SerializableSegment;
@@ -241,20 +241,20 @@ mod tests {
let mut index_writer = index.writer_with_num_threads(1).unwrap();
{
let mut doc = Document::new();
doc.add_text(&text_field, "af b");
doc.add_u32(&score_field, 3);
doc.add_text(text_field, "af b");
doc.add_u32(score_field, 3);
index_writer.add_document(doc).unwrap();
}
{
let mut doc = Document::new();
doc.add_text(&text_field, "a b c");
doc.add_u32(&score_field, 5);
doc.add_text(text_field, "a b c");
doc.add_u32(score_field, 5);
index_writer.add_document(doc).unwrap();
}
{
let mut doc = Document::new();
doc.add_text(&text_field, "a b c d");
doc.add_u32(&score_field, 7);
doc.add_text(text_field, "a b c d");
doc.add_u32(score_field, 7);
index_writer.add_document(doc).unwrap();
}
index_writer.wait().unwrap();
@@ -265,14 +265,14 @@ mod tests {
let mut index_writer = index.writer_with_num_threads(1).unwrap();
{
let mut doc = Document::new();
doc.add_text(&text_field, "af b");
doc.add_u32(&score_field, 11);
doc.add_text(text_field, "af b");
doc.add_u32(score_field, 11);
index_writer.add_document(doc).unwrap();
}
{
let mut doc = Document::new();
doc.add_text(&text_field, "a b c g");
doc.add_u32(&score_field, 13);
doc.add_text(text_field, "a b c g");
doc.add_u32(score_field, 13);
index_writer.add_document(doc).unwrap();
}
index_writer.wait().unwrap();

4
src/core/mod.rs
View File

@@ -1,13 +1,15 @@
pub mod writer;
pub mod reader;
pub mod codec;
pub mod searcher;
pub mod index;
pub mod merger;
mod segment_reader;
use std::error;
use std::io;
pub use self::segment_reader::SegmentReader;
pub fn convert_to_ioerror<E: 'static + error::Error + Send + Sync>(err: E) -> io::Error {
io::Error::new(
io::ErrorKind::InvalidData,

2
src/core/searcher.rs
View File

@@ -1,4 +1,4 @@
use core::reader::SegmentReader;
use core::SegmentReader;
use core::index::Index;
use core::index::Segment;
use DocId;

42
src/core/reader.rs → src/core/segment_reader.rs
View File

@@ -22,11 +22,6 @@ use postings::intersection;
use schema::FieldEntry;
use schema::Schema;
impl fmt::Debug for SegmentReader {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "SegmentReader({:?})", self.segment_id)
}
}
pub struct SegmentReader {
@@ -153,35 +148,8 @@ impl SegmentReader {
}
}
// impl SerializableSegment for SegmentReader {
//
// fn write_postings(&self, mut serializer: PostingsSerializer) -> io::Result<()> {
// let mut term_infos_it = self.term_infos.stream();
// loop {
// match term_infos_it.next() {
// Some((term_data, term_info)) => {
// let term = Term::from(term_data);
// try!(serializer.new_term(&term, term_info.doc_freq));
// let segment_postings = self.read_postings(term_info.postings_offset);
// try!(serializer.write_docs(&segment_postings.doc_ids[..]));
// },
// None => { break; }
// }
// }
// Ok(())
// }
//
// fn write_store(&self, )
//
// fn write(&self, mut serializer: SegmentSerializer) -> io::Result<()> {
// try!(self.write_postings(serializer.get_postings_serializer()));
// try!(self.write_store(serializer.get_store_serializer()));
//
// for doc_id in 0..self.max_doc() {
// let doc = try!(self.store_reader.get(&doc_id));
// try!(serializer.store_doc(&mut doc.text_fields()));
// }
// serializer.close()
// }
// }
impl fmt::Debug for SegmentReader {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "SegmentReader({:?})", self.segment_id)
}
}

32
src/fastfield/mod.rs
View File

@@ -45,7 +45,7 @@ mod tests {
assert_eq!(compute_num_bits(256), 9u8);
}
fn add_single_field_doc(fast_field_writers: &mut U32FastFieldsWriter, field: &Field, value: u32) {
fn add_single_field_doc(fast_field_writers: &mut U32FastFieldsWriter, field: Field, value: u32) {
let mut doc = Document::new();
doc.add_u32(field, value);
fast_field_writers.add_document(&doc);
@@ -61,9 +61,9 @@ mod tests {
let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
let mut serializer = FastFieldSerializer::new(write).unwrap();
let mut fast_field_writers = U32FastFieldsWriter::from_schema(&schema);
add_single_field_doc(&mut fast_field_writers, &field, 13u32);
add_single_field_doc(&mut fast_field_writers, &field, 14u32);
add_single_field_doc(&mut fast_field_writers, &field, 2u32);
add_single_field_doc(&mut fast_field_writers, field, 13u32);
add_single_field_doc(&mut fast_field_writers, field, 14u32);
add_single_field_doc(&mut fast_field_writers, field, 2u32);
fast_field_writers.serialize(&mut serializer).unwrap();
serializer.close().unwrap();
}
@@ -90,15 +90,15 @@ mod tests {
let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
let mut serializer = FastFieldSerializer::new(write).unwrap();
let mut fast_field_writers = U32FastFieldsWriter::from_schema(&schema);
add_single_field_doc(&mut fast_field_writers, &field, 4u32);
add_single_field_doc(&mut fast_field_writers, &field, 14_082_001u32);
add_single_field_doc(&mut fast_field_writers, &field, 3_052u32);
add_single_field_doc(&mut fast_field_writers, &field, 9002u32);
add_single_field_doc(&mut fast_field_writers, &field, 15_001u32);
add_single_field_doc(&mut fast_field_writers, &field, 777u32);
add_single_field_doc(&mut fast_field_writers, &field, 1_002u32);
add_single_field_doc(&mut fast_field_writers, &field, 1_501u32);
add_single_field_doc(&mut fast_field_writers, &field, 215u32);
add_single_field_doc(&mut fast_field_writers, field, 4u32);
add_single_field_doc(&mut fast_field_writers, field, 14_082_001u32);
add_single_field_doc(&mut fast_field_writers, field, 3_052u32);
add_single_field_doc(&mut fast_field_writers, field, 9002u32);
add_single_field_doc(&mut fast_field_writers, field, 15_001u32);
add_single_field_doc(&mut fast_field_writers, field, 777u32);
add_single_field_doc(&mut fast_field_writers, field, 1_002u32);
add_single_field_doc(&mut fast_field_writers, field, 1_501u32);
add_single_field_doc(&mut fast_field_writers, field, 215u32);
fast_field_writers.serialize(&mut serializer).unwrap();
serializer.close().unwrap();
}
@@ -142,7 +142,7 @@ mod tests {
let mut serializer = FastFieldSerializer::new(write).unwrap();
let mut fast_field_writers = U32FastFieldsWriter::from_schema(&schema);
for x in permutation.iter() {
add_single_field_doc(&mut fast_field_writers, &field, x.clone());
add_single_field_doc(&mut fast_field_writers, field, x.clone());
}
fast_field_writers.serialize(&mut serializer).unwrap();
serializer.close().unwrap();
@@ -197,7 +197,7 @@ mod tests {
let mut serializer = FastFieldSerializer::new(write).unwrap();
let mut fast_field_writers = U32FastFieldsWriter::from_schema(&schema);
for x in permutation.iter() {
add_single_field_doc(&mut fast_field_writers, &field, x.clone());
add_single_field_doc(&mut fast_field_writers, field, x.clone());
}
fast_field_writers.serialize(&mut serializer).unwrap();
serializer.close().unwrap();
@@ -229,7 +229,7 @@ mod tests {
let mut serializer = FastFieldSerializer::new(write).unwrap();
let mut fast_field_writers = U32FastFieldsWriter::from_schema(&schema);
for x in permutation.iter() {
add_single_field_doc(&mut fast_field_writers, &field, x.clone());
add_single_field_doc(&mut fast_field_writers, field, x.clone());
}
fast_field_writers.serialize(&mut serializer).unwrap();
serializer.close().unwrap();

20
src/lib.rs
View File

@@ -47,7 +47,7 @@ pub use core::searcher::Searcher;
pub use core::index::Index;
pub use schema::Term;
pub use schema::Document;
pub use core::reader::SegmentReader;
pub use core::SegmentReader;
pub use core::searcher::SegmentLocalId;
pub use self::common::TimerTree;
@@ -74,17 +74,17 @@ mod tests {
let mut index_writer = index.writer_with_num_threads(1).unwrap();
{
let mut doc = Document::new();
doc.add_text(&text_field, "af b");
doc.add_text(text_field, "af b");
index_writer.add_document(doc).unwrap();
}
{
let mut doc = Document::new();
doc.add_text(&text_field, "a b c");
doc.add_text(text_field, "a b c");
index_writer.add_document(doc).unwrap();
}
{
let mut doc = Document::new();
doc.add_text(&text_field, "a b c d");
doc.add_text(text_field, "a b c d");
index_writer.add_document(doc).unwrap();
}
assert!(index_writer.wait().is_ok());
@@ -108,17 +108,17 @@ mod tests {
let mut index_writer = index.writer_with_num_threads(1).unwrap();
{
let mut doc = Document::new();
doc.add_text(&text_field, "af b");
doc.add_text(text_field, "af b");
index_writer.add_document(doc).unwrap();
}
{
let mut doc = Document::new();
doc.add_text(&text_field, "a b c");
doc.add_text(text_field, "a b c");
index_writer.add_document(doc).unwrap();
}
{
let mut doc = Document::new();
doc.add_text(&text_field, "a b c d");
doc.add_text(text_field, "a b c d");
index_writer.add_document(doc).unwrap();
}
index_writer.wait().unwrap();
@@ -176,17 +176,17 @@ mod tests {
let mut index_writer = index.writer_with_num_threads(1).unwrap();
{
let mut doc = Document::new();
doc.add_text(&text_field, "af b");
doc.add_text(text_field, "af b");
index_writer.add_document(doc).unwrap();
}
{
let mut doc = Document::new();
doc.add_text(&text_field, "a b c");
doc.add_text(text_field, "a b c");
index_writer.add_document(doc).unwrap();
}
{
let mut doc = Document::new();
doc.add_text(&text_field, "a b c d");
doc.add_text(text_field, "a b c d");
index_writer.add_document(doc).unwrap();
}
index_writer.wait().unwrap();

2
src/query/multi_term_query.rs
View File

@@ -6,7 +6,7 @@ use std::io;
use core::searcher::Searcher;
use collector::Collector;
use core::searcher::SegmentLocalId;
use core::reader::SegmentReader;
use core::SegmentReader;
use postings::Postings;
use postings::SegmentPostings;
use postings::intersection;

25
src/schema/document.rs
View File

@@ -21,11 +21,11 @@ impl Document {
self.field_values.len()
}
pub fn add_text(&mut self, field: &Field, text: &str) {
pub fn add_text(&mut self, field: Field, text: &str) {
self.add(FieldValue::Text(field.clone(), String::from(text)));
}
pub fn add_u32(&mut self, field: &Field, value: u32) {
pub fn add_u32(&mut self, field: Field, value: u32) {
self.add(FieldValue::U32(field.clone(), value));
}
@@ -59,3 +59,24 @@ impl From<Vec<FieldValue>> for Document {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use schema::Schema;
use schema::TEXT;
#[test]
fn test_doc() {
let mut schema = Schema::new();
let text_field = schema.add_text_field("title", TEXT);
let mut doc = Document::new();
doc.add_text(text_field, "My title");
assert_eq!(doc.get_fields().len(), 1);
}
}

20
src/schema/field_entry.rs
View File

@@ -11,6 +11,7 @@ pub enum FieldEntry {
}
impl FieldEntry {
pub fn get_field_name(&self,) -> &str {
match self {
&FieldEntry::Text(ref field_name, _) => {
@@ -44,4 +45,21 @@ impl FieldEntry {
}
}
}
}
}
// TODO implement a nicer JSON format
#[cfg(test)]
mod tests {
use super::*;
use schema::TEXT;
use rustc_serialize::json;
#[test]
fn test_json_serialization() {
let field_value = FieldEntry::Text(String::from("title"), TEXT);
assert_eq!(format!("{}", json::as_json(&field_value)),
"{\"variant\":\"Text\",\"fields\":[\"title\",{\"indexing_options\":\"TokenizedWithFreqAndPosition\",\"stored\":false,\"fast\":false}]}" );
}
}

2
src/schema/schema.rs
View File

@@ -131,6 +131,4 @@ impl Schema {
field
}
}