From 1807e0710c2b89402476660f233b841aeef9a0bd Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Sun, 14 Aug 2016 10:58:04 +0900 Subject: [PATCH] Bugfix in schema json decoder --- TODO.md | 1 - docs/tutorial.md | 40 ++++++++++++++---------------- src/cli/commands/new.rs | 4 +-- src/query/daat_multiterm_scorer.rs | 20 +++++++-------- src/query/multi_term_query.rs | 25 +++++++++---------- src/schema/field_entry.rs | 29 +++++++++++----------- 6 files changed, 57 insertions(+), 62 deletions(-) diff --git a/TODO.md b/TODO.md index 1534388a0..a8e515e80 100644 --- a/TODO.md +++ b/TODO.md @@ -1,7 +1,6 @@ position not stored lenient mode for query parser phrase queries -masks for union documentation query explain with proper term names better schema JSON diff --git a/docs/tutorial.md b/docs/tutorial.md index 71675bcf4..a59d7738a 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -40,6 +40,7 @@ the definition of the schema of our new index. When asked answer to the question as follows: ```none + Creating new index Let's define it's schema! @@ -56,7 +57,7 @@ When asked answer to the question as follows: - New field name ? body + New field name ? body Text or unsigned 32-bit Integer (T/I) ? T Should the field be stored (Y/N) ? Y Should the field be indexed (Y/N) ? Y @@ -75,37 +76,32 @@ When asked answer to the question as follows: [ { - "variant": "Text", - "fields": [ - "title", - { - "indexing_options": "TokenizedWithFreqAndPosition", - "stored": true + "name": "title", + "type": "text", + "options": { + "indexing": "position", + "stored": true } - ] }, { - "variant": "Text", - "fields": [ - "body", - { - "indexing_options": "TokenizedWithFreqAndPosition", - "stored": true + "name": "body", + "type": "text", + "options": { + "indexing": "position", + "stored": true } - ] }, { - "variant": "Text", - "fields": [ - "url", - { - "indexing_options": "Unindexed", - "stored": true + "name": "url", + "type": "text", + "options": { + "indexing": "unindexed", + "stored": true } - ] } ] + ``` If you want to know more about the meaning of these options, you can check out the [schema doc page](http://fulmicoton.com/tantivy/tantivy/schema/index.html). diff --git a/src/cli/commands/new.rs b/src/cli/commands/new.rs index 8454eb825..64ba9201b 100644 --- a/src/cli/commands/new.rs +++ b/src/cli/commands/new.rs @@ -6,7 +6,7 @@ use tantivy::schema::*; use tantivy::Index; use std::io; use ansi_term::Style; -use ansi_term::Colour::{Red, Blue, Green, Black}; +use ansi_term::Colour::{Red, Blue, Green}; use std::io::Write; use std::ascii::AsciiExt; use rustc_serialize::json; @@ -141,7 +141,7 @@ fn run_new(directory: PathBuf) -> tantivy::Result<()> { } } let schema_json = format!("{}", json::as_pretty_json(&schema)); - println!("\n{}\n", Style::new().on(Blue).fg(Black).paint(schema_json)); + println!("\n{}\n", Style::new().fg(Green).paint(schema_json)); let mut index = try!(Index::create(&directory, schema)); index.save_metas() } diff --git a/src/query/daat_multiterm_scorer.rs b/src/query/daat_multiterm_scorer.rs index 60b132579..65a6ba78a 100644 --- a/src/query/daat_multiterm_scorer.rs +++ b/src/query/daat_multiterm_scorer.rs @@ -229,27 +229,27 @@ mod tests { } #[test] - pub fn test_union_postings() { + pub fn test_daat_scorer() { let left_fieldnorms = create_u32_fastfieldreader(Field(1), vec!(100,200,300)); let right_fieldnorms = create_u32_fastfieldreader(Field(2), vec!(15,25,35)); let left = VecPostings::from(vec!(1, 2, 3)); let right = VecPostings::from(vec!(1, 3, 8)); let tfidf = TfIdf::new(vec!(0f32, 1f32, 2f32), vec!(1f32, 4f32)); - let mut union = DAATMultiTermScorer::new( + let mut daat_scorer = DAATMultiTermScorer::new( vec!( (Occur::Should, left, left_fieldnorms), (Occur::Should, right, right_fieldnorms), ), tfidf ); - assert_eq!(union.next(), Some(1u32)); - assert!(abs_diff(union.score(), 2.182179f32) < 0.001); - assert_eq!(union.next(), Some(2u32)); - assert!(abs_diff(union.score(), 0.2236068) < 0.001f32); - assert_eq!(union.next(), Some(3u32)); - assert_eq!(union.next(), Some(8u32)); - assert!(abs_diff(union.score(), 0.8944272f32) < 0.001f32); - assert!(!union.advance()); + assert_eq!(daat_scorer.next(), Some(1u32)); + assert!(abs_diff(daat_scorer.score(), 2.182179f32) < 0.001); + assert_eq!(daat_scorer.next(), Some(2u32)); + assert!(abs_diff(daat_scorer.score(), 0.2236068) < 0.001f32); + assert_eq!(daat_scorer.next(), Some(3u32)); + assert_eq!(daat_scorer.next(), Some(8u32)); + assert!(abs_diff(daat_scorer.score(), 0.8944272f32) < 0.001f32); + assert!(!daat_scorer.advance()); } } diff --git a/src/query/multi_term_query.rs b/src/query/multi_term_query.rs index 48da22d1e..9903fc9e1 100644 --- a/src/query/multi_term_query.rs +++ b/src/query/multi_term_query.rs @@ -35,7 +35,7 @@ impl MultiTermQuery { self.occur_terms.len() } - fn scorer(&self, searcher: &Searcher) -> TfIdf { + fn similitude(&self, searcher: &Searcher) -> TfIdf { let num_terms = self.num_terms(); let num_docs = searcher.num_docs() as f32; let idfs: Vec = self.occur_terms @@ -58,16 +58,16 @@ impl MultiTermQuery { .iter() .map(|&(_, ref term)| format!("{:?}", &term)) .collect(); - let mut tfidf_scorer = TfIdf::new(query_coords, idfs); - tfidf_scorer.set_term_names(term_names); - tfidf_scorer + let mut tfidf = TfIdf::new(query_coords, idfs); + tfidf.set_term_names(term_names); + tfidf } - fn search_segment<'a, 'b, TScorer: MultiTermAccumulator>( + fn search_segment<'a, 'b, TAccumulator: MultiTermAccumulator>( &'b self, reader: &'b SegmentReader, - multi_term_scorer: TScorer, - mut timer: OpenTimer<'a>) -> Result> { + accumulator: TAccumulator, + mut timer: OpenTimer<'a>) -> Result> { let mut postings_and_fieldnorms = Vec::with_capacity(self.num_terms()); { let mut decode_timer = timer.open("decode_all"); @@ -87,7 +87,7 @@ impl MultiTermQuery { // TODO putting the SHOULD at the end of the list should push the limit. return Err(Error::InvalidArgument(String::from("Limit of 64 terms was exceeded."))); } - Ok(DAATMultiTermScorer::new(postings_and_fieldnorms, multi_term_scorer)) + Ok(DAATMultiTermScorer::new(postings_and_fieldnorms, accumulator)) } } @@ -119,12 +119,12 @@ impl Query for MultiTermQuery { searcher: &Searcher, doc_address: &DocAddress) -> Result { let segment_reader = &searcher.segments()[doc_address.segment_ord() as usize]; - let multi_term_scorer = SimilarityExplainer::from(self.scorer(searcher)); + let similitude = SimilarityExplainer::from(self.similitude(searcher)); let mut timer_tree = TimerTree::new(); let mut postings = try!( self.search_segment( segment_reader, - multi_term_scorer, + similitude, timer_tree.open("explain")) ); Ok(match postings.skip_next(doc_address.doc()) { @@ -144,8 +144,7 @@ impl Query for MultiTermQuery { &self, searcher: &Searcher, collector: &mut C) -> Result { - let mut timer_tree = TimerTree::new(); - let multi_term_scorer = self.scorer(searcher); + let mut timer_tree = TimerTree::new(); { let mut search_timer = timer_tree.open("search"); for (segment_ord, segment_reader) in searcher.segments().iter().enumerate() { @@ -157,7 +156,7 @@ impl Query for MultiTermQuery { let mut postings = try!( self.search_segment( segment_reader, - multi_term_scorer.clone(), + self.similitude(searcher), segment_search_timer.open("get_postings")) ); { diff --git a/src/schema/field_entry.rs b/src/schema/field_entry.rs index 6e2c3c03e..9e5cf14f0 100644 --- a/src/schema/field_entry.rs +++ b/src/schema/field_entry.rs @@ -55,24 +55,25 @@ impl Decodable for FieldEntry { let name = try!(d.read_struct_field("name", 0, |d| { d.read_str() })); - let field_type: String = try!(d.read_struct_field("type", 0, |d| { + let field_type: String = try!(d.read_struct_field("type", 1, |d| { d.read_str() })); - match field_type.as_ref() { - "u32" => { - let u32_options = try!(U32Options::decode(d)); - Ok(FieldEntry::new_u32(name, u32_options)) + d.read_struct_field("options", 2, |d| { + match field_type.as_ref() { + "u32" => { + let u32_options = try!(U32Options::decode(d)); + Ok(FieldEntry::new_u32(name, u32_options)) + } + "text" => { + let text_options = try!(TextOptions::decode(d)); + Ok(FieldEntry::new_text(name, text_options)) + } + _ => { + Err(d.error(&format!("Field type {:?} unknown", field_type))) + } } - "text" => { - let text_options = try!(TextOptions::decode(d)); - Ok(FieldEntry::new_text(name, text_options)) - } - _ => { - Err(d.error(&format!("Field type {:?} unknown", field_type))) - } - } + }) }) - } }