Bump version

Closes #731

CHANGELOG.md

@@ -1,3 +1,12 @@
+Tantivy 0.11.3
+=======================
+- Fixed DateTime as a fast field (#735)
+
+Tantivy 0.11.2
+=======================
+- The future returned by `IndexWriter::merge` does not borrow `self` mutably anymore (#732)
+- Exposing a constructor for `WatchHandle` (#731)
+
 Tantivy 0.11.1
 =====================
 - Bug fix #729

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tantivy"
-version = "0.11.1"
+version = "0.11.3"
 authors = ["Paul Masurel <paul.masurel@gmail.com>"]
 license = "MIT"
 categories = ["database-implementations", "data-structures"]

src/directory/directory.rs

@@ -119,7 +119,7 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     /// Specifically, subsequent writes or flushes should
     /// have no effect on the returned `ReadOnlySource` object.
     ///
-    /// You should only use this to read files create with [`open_write`]
+    /// You should only use this to read files create with [Directory::open_write].
     fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, OpenReadError>;
 
     /// Removes a file
@@ -160,7 +160,7 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     ///
     /// This should only be used for small files.
     ///
-    /// You should only use this to read files create with [`atomic_write`]
+    /// You should only use this to read files create with [Directory::atomic_write].
     fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, OpenReadError>;
 
     /// Atomically replace the content of a file with data.
@@ -197,7 +197,7 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
     /// Registers a callback that will be called whenever a change on the `meta.json`
     /// using the `atomic_write` API is detected.
     ///
-    /// The behavior when using `.watch()` on a file using `.open_write(...)` is, on the other
+    /// The behavior when using `.watch()` on a file using [Directory::open_write] is, on the other
     /// hand, undefined.
     ///
     /// The file will be watched for the lifetime of the returned `WatchHandle`. The caller is

src/directory/mod.rs

@@ -23,11 +23,9 @@ pub use self::directory::{Directory, DirectoryClone};
 pub use self::directory_lock::{Lock, INDEX_WRITER_LOCK, META_LOCK};
 pub use self::ram_directory::RAMDirectory;
 pub use self::read_only_source::ReadOnlySource;
-pub(crate) use self::watch_event_router::WatchCallbackList;
-pub use self::watch_event_router::{WatchCallback, WatchHandle};
+pub use self::watch_event_router::{WatchCallback, WatchCallbackList, WatchHandle};
 use std::io::{self, BufWriter, Write};
 use std::path::PathBuf;
-
 /// Outcome of the Garbage collection
 pub struct GarbageCollectionResult {
     /// List of files that were deleted in this cycle

src/directory/watch_event_router.rs

@@ -24,13 +24,20 @@ pub struct WatchCallbackList {
 #[derive(Clone)]
 pub struct WatchHandle(Arc<WatchCallback>);
 
+impl WatchHandle {
+    /// Create a WatchHandle handle.
+    pub fn new(watch_callback: Arc<WatchCallback>) -> WatchHandle {
+        WatchHandle(watch_callback)
+    }
+}
+
 impl WatchCallbackList {
     /// Suscribes a new callback and returns a handle that controls the lifetime of the callback.
     pub fn subscribe(&self, watch_callback: WatchCallback) -> WatchHandle {
         let watch_callback_arc = Arc::new(watch_callback);
         let watch_callback_weak = Arc::downgrade(&watch_callback_arc);
         self.router.write().unwrap().push(watch_callback_weak);
-        WatchHandle(watch_callback_arc)
+        WatchHandle::new(watch_callback_arc)
     }
 
     fn list_callback(&self) -> Vec<Arc<WatchCallback>> {

src/fastfield/mod.rs

@@ -33,6 +33,7 @@ pub use self::reader::FastFieldReader;
 pub use self::readers::FastFieldReaders;
 pub use self::serializer::FastFieldSerializer;
 pub use self::writer::{FastFieldsWriter, IntFastFieldWriter};
+use crate::chrono::{NaiveDateTime, Utc};
 use crate::common;
 use crate::schema::Cardinality;
 use crate::schema::FieldType;
@@ -49,7 +50,7 @@ mod serializer;
 mod writer;
 
 /// Trait for types that are allowed for fast fields: (u64, i64 and f64).
-pub trait FastValue: Default + Clone + Copy + Send + Sync + PartialOrd {
+pub trait FastValue: Clone + Copy + Send + Sync + PartialOrd {
     /// Converts a value from u64
     ///
     /// Internally all fast field values are encoded as u64.
@@ -69,6 +70,12 @@ pub trait FastValue: Default + Clone + Copy + Send + Sync + PartialOrd {
     /// Cast value to `u64`.
     /// The value is just reinterpreted in memory.
     fn as_u64(&self) -> u64;
+
+    /// Build a default value. This default value is never used, so the value does not
+    /// really matter.
+    fn make_zero() -> Self {
+        Self::from_u64(0i64.to_u64())
+    }
 }
 
 impl FastValue for u64 {
@@ -135,11 +142,34 @@ impl FastValue for f64 {
     }
 }
 
+impl FastValue for crate::DateTime {
+    fn from_u64(timestamp_u64: u64) -> Self {
+        let timestamp_i64 = i64::from_u64(timestamp_u64);
+        crate::DateTime::from_utc(NaiveDateTime::from_timestamp(timestamp_i64, 0), Utc)
+    }
+
+    fn to_u64(&self) -> u64 {
+        self.timestamp().to_u64()
+    }
+
+    fn fast_field_cardinality(field_type: &FieldType) -> Option<Cardinality> {
+        match *field_type {
+            FieldType::Date(ref integer_options) => integer_options.get_fastfield_cardinality(),
+            _ => None,
+        }
+    }
+
+    fn as_u64(&self) -> u64 {
+        self.timestamp().as_u64()
+    }
+}
+
 fn value_to_u64(value: &Value) -> u64 {
     match *value {
         Value::U64(ref val) => *val,
         Value::I64(ref val) => common::i64_to_u64(*val),
         Value::F64(ref val) => common::f64_to_u64(*val),
+        Value::Date(ref datetime) => common::i64_to_u64(datetime.timestamp()),
         _ => panic!("Expected a u64/i64/f64 field, got {:?} ", value),
     }
 }
@@ -151,10 +181,12 @@ mod tests {
     use crate::common::CompositeFile;
     use crate::directory::{Directory, RAMDirectory, WritePtr};
     use crate::fastfield::FastFieldReader;
-    use crate::schema::Document;
+    use crate::merge_policy::NoMergePolicy;
     use crate::schema::Field;
     use crate::schema::Schema;
     use crate::schema::FAST;
+    use crate::schema::{Document, IntOptions};
+    use crate::{Index, SegmentId, SegmentReader};
     use once_cell::sync::Lazy;
     use rand::prelude::SliceRandom;
     use rand::rngs::StdRng;
@@ -178,6 +210,12 @@ mod tests {
         assert_eq!(test_fastfield.get(2), 300);
     }
 
+    #[test]
+    pub fn test_fastfield_i64_u64() {
+        let datetime = crate::DateTime::from_utc(NaiveDateTime::from_timestamp(0i64, 0), Utc);
+        assert_eq!(i64::from_u64(datetime.to_u64()), 0i64);
+    }
+
     #[test]
     fn test_intfastfield_small() {
         let path = Path::new("test");
@@ -429,6 +467,93 @@ mod tests {
             }
         }
     }
+
+    #[test]
+    fn test_merge_missing_date_fast_field() {
+        let mut schema_builder = Schema::builder();
+        let date_field = schema_builder.add_date_field("date", FAST);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        index_writer.set_merge_policy(Box::new(NoMergePolicy));
+        index_writer.add_document(doc!(date_field =>crate::chrono::prelude::Utc::now()));
+        index_writer.commit().unwrap();
+        index_writer.add_document(doc!());
+        index_writer.commit().unwrap();
+        let reader = index.reader().unwrap();
+        let segment_ids: Vec<SegmentId> = reader
+            .searcher()
+            .segment_readers()
+            .iter()
+            .map(SegmentReader::segment_id)
+            .collect();
+        assert_eq!(segment_ids.len(), 2);
+        let merge_future = index_writer.merge(&segment_ids[..]);
+        let merge_res = futures::executor::block_on(merge_future);
+        assert!(merge_res.is_ok());
+        assert!(reader.reload().is_ok());
+        assert_eq!(reader.searcher().segment_readers().len(), 1);
+    }
+
+    #[test]
+    fn test_default_datetime() {
+        assert_eq!(crate::DateTime::make_zero().timestamp(), 0i64);
+    }
+
+    #[test]
+    fn test_datefastfield() {
+        use crate::fastfield::FastValue;
+        let mut schema_builder = Schema::builder();
+        let date_field = schema_builder.add_date_field("date", FAST);
+        let multi_date_field = schema_builder.add_date_field(
+            "multi_date",
+            IntOptions::default().set_fast(Cardinality::MultiValues),
+        );
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
+        index_writer.set_merge_policy(Box::new(NoMergePolicy));
+        index_writer.add_document(doc!(
+            date_field => crate::DateTime::from_u64(1i64.to_u64()),
+            multi_date_field => crate::DateTime::from_u64(2i64.to_u64()),
+            multi_date_field => crate::DateTime::from_u64(3i64.to_u64())
+        ));
+        index_writer.add_document(doc!(
+            date_field => crate::DateTime::from_u64(4i64.to_u64())
+        ));
+        index_writer.add_document(doc!(
+            multi_date_field => crate::DateTime::from_u64(5i64.to_u64()),
+            multi_date_field => crate::DateTime::from_u64(6i64.to_u64())
+        ));
+        index_writer.commit().unwrap();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
+        assert_eq!(searcher.segment_readers().len(), 1);
+        let segment_reader = searcher.segment_reader(0);
+        let fast_fields = segment_reader.fast_fields();
+        let date_fast_field = fast_fields.date(date_field).unwrap();
+        let dates_fast_field = fast_fields.dates(multi_date_field).unwrap();
+        let mut dates = vec![];
+        {
+            assert_eq!(date_fast_field.get(0u32).timestamp(), 1i64);
+            dates_fast_field.get_vals(0u32, &mut dates);
+            assert_eq!(dates.len(), 2);
+            assert_eq!(dates[0].timestamp(), 2i64);
+            assert_eq!(dates[1].timestamp(), 3i64);
+        }
+        {
+            assert_eq!(date_fast_field.get(1u32).timestamp(), 4i64);
+            dates_fast_field.get_vals(1u32, &mut dates);
+            assert!(dates.is_empty());
+        }
+        {
+            assert_eq!(date_fast_field.get(2u32).timestamp(), 0i64);
+            dates_fast_field.get_vals(2u32, &mut dates);
+            assert_eq!(dates.len(), 2);
+            assert_eq!(dates[0].timestamp(), 5i64);
+            assert_eq!(dates[1].timestamp(), 6i64);
+        }
+    }
 }
 
 #[cfg(all(test, feature = "unstable"))]

src/fastfield/multivalued/reader.rs

@@ -45,7 +45,7 @@ impl<Item: FastValue> MultiValueIntFastFieldReader<Item> {
     pub fn get_vals(&self, doc: DocId, vals: &mut Vec<Item>) {
         let (start, stop) = self.range(doc);
         let len = (stop - start) as usize;
-        vals.resize(len, Item::default());
+        vals.resize(len, Item::make_zero());
         self.vals_reader.get_range_u64(start, &mut vals[..]);
     }
 

src/fastfield/readers.rs

@@ -15,9 +15,11 @@ pub struct FastFieldReaders {
     fast_field_i64: HashMap<Field, FastFieldReader<i64>>,
     fast_field_u64: HashMap<Field, FastFieldReader<u64>>,
     fast_field_f64: HashMap<Field, FastFieldReader<f64>>,
+    fast_field_date: HashMap<Field, FastFieldReader<crate::DateTime>>,
     fast_field_i64s: HashMap<Field, MultiValueIntFastFieldReader<i64>>,
     fast_field_u64s: HashMap<Field, MultiValueIntFastFieldReader<u64>>,
     fast_field_f64s: HashMap<Field, MultiValueIntFastFieldReader<f64>>,
+    fast_field_dates: HashMap<Field, MultiValueIntFastFieldReader<crate::DateTime>>,
     fast_bytes: HashMap<Field, BytesFastFieldReader>,
     fast_fields_composite: CompositeFile,
 }
@@ -26,6 +28,7 @@ enum FastType {
     I64,
     U64,
     F64,
+    Date,
 }
 
 fn type_and_cardinality(field_type: &FieldType) -> Option<(FastType, Cardinality)> {
@@ -39,6 +42,9 @@ fn type_and_cardinality(field_type: &FieldType) -> Option<(FastType, Cardinality
         FieldType::F64(options) => options
             .get_fastfield_cardinality()
             .map(|cardinality| (FastType::F64, cardinality)),
+        FieldType::Date(options) => options
+            .get_fastfield_cardinality()
+            .map(|cardinality| (FastType::Date, cardinality)),
         FieldType::HierarchicalFacet => Some((FastType::U64, Cardinality::MultiValues)),
         _ => None,
     }
@@ -53,9 +59,11 @@ impl FastFieldReaders {
             fast_field_i64: Default::default(),
             fast_field_u64: Default::default(),
             fast_field_f64: Default::default(),
+            fast_field_date: Default::default(),
             fast_field_i64s: Default::default(),
             fast_field_u64s: Default::default(),
             fast_field_f64s: Default::default(),
+            fast_field_dates: Default::default(),
             fast_bytes: Default::default(),
             fast_fields_composite: fast_fields_composite.clone(),
         };
@@ -95,6 +103,12 @@ impl FastFieldReaders {
                                         FastFieldReader::open(fast_field_data.clone()),
                                     );
                                 }
+                                FastType::Date => {
+                                    fast_field_readers.fast_field_date.insert(
+                                        field,
+                                        FastFieldReader::open(fast_field_data.clone()),
+                                    );
+                                }
                             }
                         } else {
                             return Err(From::from(FastFieldNotAvailableError::new(field_entry)));
@@ -130,6 +144,14 @@ impl FastFieldReaders {
                                         .fast_field_f64s
                                         .insert(field, multivalued_int_fast_field);
                                 }
+                                FastType::Date => {
+                                    let vals_reader = FastFieldReader::open(fast_field_data);
+                                    let multivalued_int_fast_field =
+                                        MultiValueIntFastFieldReader::open(idx_reader, vals_reader);
+                                    fast_field_readers
+                                        .fast_field_dates
+                                        .insert(field, multivalued_int_fast_field);
+                                }
                             }
                         } else {
                             return Err(From::from(FastFieldNotAvailableError::new(field_entry)));
@@ -167,6 +189,9 @@ impl FastFieldReaders {
         if let Some(f64_ff_reader) = self.f64(field) {
             return Some(f64_ff_reader.into_u64_reader());
         }
+        if let Some(date_ff_reader) = self.date(field) {
+            return Some(date_ff_reader.into_u64_reader());
+        }
         None
     }
 
@@ -177,6 +202,13 @@ impl FastFieldReaders {
         self.fast_field_i64.get(&field).cloned()
     }
 
+    /// Returns the `i64` fast field reader reader associated to `field`.
+    ///
+    /// If `field` is not a i64 fast field, this method returns `None`.
+    pub fn date(&self, field: Field) -> Option<FastFieldReader<crate::DateTime>> {
+        self.fast_field_date.get(&field).cloned()
+    }
+
     /// Returns the `f64` fast field reader reader associated to `field`.
     ///
     /// If `field` is not a f64 fast field, this method returns `None`.
@@ -223,6 +255,13 @@ impl FastFieldReaders {
         self.fast_field_f64s.get(&field).cloned()
     }
 
+    /// Returns a `crate::DateTime` multi-valued fast field reader reader associated to `field`.
+    ///
+    /// If `field` is not a `crate::DateTime` multi-valued fast field, this method returns `None`.
+    pub fn dates(&self, field: Field) -> Option<MultiValueIntFastFieldReader<crate::DateTime>> {
+        self.fast_field_dates.get(&field).cloned()
+    }
+
     /// Returns the `bytes` fast field reader associated to `field`.
     ///
     /// If `field` is not a bytes fast field, returns `None`.

src/fastfield/writer.rs

@@ -4,7 +4,7 @@ use crate::common::BinarySerializable;
 use crate::common::VInt;
 use crate::fastfield::{BytesFastFieldWriter, FastFieldSerializer};
 use crate::postings::UnorderedTermId;
-use crate::schema::{Cardinality, Document, Field, FieldType, Schema};
+use crate::schema::{Cardinality, Document, Field, FieldEntry, FieldType, Schema};
 use crate::termdict::TermOrdinal;
 use fnv::FnvHashMap;
 use std::collections::HashMap;
@@ -17,6 +17,14 @@ pub struct FastFieldsWriter {
     bytes_value_writers: Vec<BytesFastFieldWriter>,
 }
 
+fn fast_field_default_value(field_entry: &FieldEntry) -> u64 {
+    match *field_entry.field_type() {
+        FieldType::I64(_) | FieldType::Date(_) => common::i64_to_u64(0i64),
+        FieldType::F64(_) => common::f64_to_u64(0.0f64),
+        _ => 0u64,
+    }
+}
+
 impl FastFieldsWriter {
     /// Create all `FastFieldWriter` required by the schema.
     pub fn from_schema(schema: &Schema) -> FastFieldsWriter {
@@ -25,18 +33,15 @@ impl FastFieldsWriter {
         let mut bytes_value_writers = Vec::new();
 
         for (field, field_entry) in schema.fields() {
-            let default_value = match *field_entry.field_type() {
-                FieldType::I64(_) => common::i64_to_u64(0i64),
-                FieldType::F64(_) => common::f64_to_u64(0.0f64),
-                _ => 0u64,
-            };
             match *field_entry.field_type() {
                 FieldType::I64(ref int_options)
                 | FieldType::U64(ref int_options)
-                | FieldType::F64(ref int_options) => {
+                | FieldType::F64(ref int_options)
+                | FieldType::Date(ref int_options) => {
                     match int_options.get_fastfield_cardinality() {
                         Some(Cardinality::SingleValue) => {
                             let mut fast_field_writer = IntFastFieldWriter::new(field);
+                            let default_value = fast_field_default_value(field_entry);
                             fast_field_writer.set_val_if_missing(default_value);
                             single_value_writers.push(fast_field_writer);
                         }

src/indexer/index_writer.rs

@@ -514,9 +514,13 @@ impl IndexWriter {
     /// Merges a given list of segments
     ///
     /// `segment_ids` is required to be non-empty.
-    pub async fn merge(&mut self, segment_ids: &[SegmentId]) -> crate::Result<SegmentMeta> {
+    pub fn merge(
+        &mut self,
+        segment_ids: &[SegmentId],
+    ) -> impl Future<Output = crate::Result<SegmentMeta>> {
         let merge_operation = self.segment_updater.make_merge_operation(segment_ids);
-        self.segment_updater.start_merge(merge_operation)?.await
+        let segment_updater = self.segment_updater.clone();
+        async move { segment_updater.start_merge(merge_operation)?.await }
     }
 
     /// Closes the current document channel send.

src/indexer/mod.rs

@@ -33,6 +33,7 @@ pub type DefaultMergePolicy = LogMergePolicy;
 mod tests {
     use crate::schema::{self, Schema};
     use crate::{Index, Term};
+
     #[test]
     fn test_advance_delete_bug() {
         let mut schema_builder = Schema::builder();

src/schema/value.rs

@@ -209,8 +209,8 @@ impl From<f64> for Value {
     }
 }
 
-impl From<DateTime> for Value {
-    fn from(date_time: DateTime) -> Value {
+impl From<crate::DateTime> for Value {
+    fn from(date_time: crate::DateTime) -> Value {
         Value::Date(date_time)
     }
 }