Merge branch 'issue/43'

Conflicts:
	src/directory/mmap_directory.rs

Cargo.toml

@@ -24,17 +24,23 @@ rustc-serialize = "0.3"
 log = "0.3.6"
 combine = "2.2"
 tempdir = "0.3"
+
+
 bincode = "0.5"
 libc = {version = "0.2.20", optional=true}
 num_cpus = "1.2"
 itertools = "0.5.9"
 lz4 = "1.20"
+bit-set = "0.4.0"
 time = "0.1"
 uuid = { version = "0.4", features = ["v4", "rustc-serialize"] }
 chan = "0.1"
 version = "2"
 crossbeam = "0.2"
 
+futures = "0.1.9"
+futures-cpupool = "0.1.2"
+
 [dev-dependencies]
 rand = "0.3"
 

src/collector/chained_collector.rs

@@ -1,7 +1,7 @@
+use Result;
 use collector::Collector;
 use SegmentLocalId;
 use SegmentReader;
-use std::io;
 use DocId;
 use Score;
 
@@ -12,7 +12,7 @@ use Score;
 pub struct DoNothingCollector;
 impl Collector for DoNothingCollector {
     #[inline]
-    fn set_segment(&mut self, _: SegmentLocalId, _: &SegmentReader) -> io::Result<()> {
+    fn set_segment(&mut self, _: SegmentLocalId, _: &SegmentReader) -> Result<()> {
         Ok(())
     }
     #[inline]
@@ -38,7 +38,7 @@ impl<Left: Collector, Right: Collector> ChainedCollector<Left, Right> {
 }
 
 impl<Left: Collector, Right: Collector> Collector for ChainedCollector<Left, Right> {
-    fn set_segment(&mut self, segment_local_id: SegmentLocalId, segment: &SegmentReader) -> io::Result<()> {
+    fn set_segment(&mut self, segment_local_id: SegmentLocalId, segment: &SegmentReader) -> Result<()> {
         try!(self.left.set_segment(segment_local_id, segment));
         try!(self.right.set_segment(segment_local_id, segment));
         Ok(())

src/collector/count_collector.rs

@@ -1,7 +1,7 @@
-use std::io;
 use super::Collector;
 use DocId;
 use Score;
+use Result;
 use SegmentReader;
 use SegmentLocalId;
 
@@ -28,7 +28,7 @@ impl Default for CountCollector {
 
 impl Collector for CountCollector {
 
-    fn set_segment(&mut self, _: SegmentLocalId, _: &SegmentReader) -> io::Result<()> {
+    fn set_segment(&mut self, _: SegmentLocalId, _: &SegmentReader) -> Result<()> {
         Ok(())
     }
 

src/collector/mod.rs

@@ -2,7 +2,7 @@ use SegmentReader;
 use SegmentLocalId;
 use DocId;
 use Score;
-use std::io;
+use Result;
 
 mod count_collector;
 pub use self::count_collector::CountCollector;
@@ -48,14 +48,14 @@ pub use self::chained_collector::chain;
 pub trait Collector {
     /// `set_segment` is called before beginning to enumerate 
     /// on this segment.
-    fn set_segment(&mut self, segment_local_id: SegmentLocalId, segment: &SegmentReader) -> io::Result<()>;
+    fn set_segment(&mut self, segment_local_id: SegmentLocalId, segment: &SegmentReader) -> Result<()>;
     /// The query pushes the scored document to the collector via this method.
     fn collect(&mut self, doc: DocId, score: Score);
 }
 
 
 impl<'a, C: Collector> Collector for &'a mut C {
-    fn set_segment(&mut self, segment_local_id: SegmentLocalId, segment: &SegmentReader) -> io::Result<()> {
+    fn set_segment(&mut self, segment_local_id: SegmentLocalId, segment: &SegmentReader) -> Result<()> {
         (*self).set_segment(segment_local_id, segment)
     }
     /// The query pushes the scored document to the collector via this method.
@@ -73,7 +73,6 @@ pub mod tests {
     use DocId;
     use Score;
     use core::SegmentReader;
-    use std::io;
     use SegmentLocalId;
     use fastfield::U32FastFieldReader;
     use schema::Field;
@@ -107,7 +106,7 @@ pub mod tests {
 
     impl Collector for TestCollector {
 
-        fn set_segment(&mut self, _: SegmentLocalId, reader: &SegmentReader) -> io::Result<()> {
+        fn set_segment(&mut self, _: SegmentLocalId, reader: &SegmentReader) -> Result<()> {
             self.offset += self.segment_max_doc;
             self.segment_max_doc = reader.max_doc();
             Ok(())
@@ -140,13 +139,13 @@ pub mod tests {
             }
         }
 
-        pub fn vals(&self,) -> &Vec<u32> {
-            &self.vals
+        pub fn vals(self,) -> Vec<u32> {
+            self.vals
         }
     }
         
     impl Collector for FastFieldTestCollector {
-        fn set_segment(&mut self, _: SegmentLocalId, reader: &SegmentReader) -> io::Result<()> {
+        fn set_segment(&mut self, _: SegmentLocalId, reader: &SegmentReader) -> Result<()> {
             self.ff_reader = Some(try!(reader.get_fast_field_reader(self.field)));
             Ok(())
         }

src/collector/multi_collector.rs

@@ -1,7 +1,7 @@
-use std::io;
 use super::Collector;
 use DocId;
 use Score;
+use Result;
 use SegmentReader;
 use SegmentLocalId;
 
@@ -25,7 +25,7 @@ impl<'a> MultiCollector<'a> {
 
 
 impl<'a> Collector for MultiCollector<'a> {
-    fn set_segment(&mut self, segment_local_id: SegmentLocalId, segment: &SegmentReader) -> io::Result<()> {
+    fn set_segment(&mut self, segment_local_id: SegmentLocalId, segment: &SegmentReader) -> Result<()> {
         for collector in &mut self.collectors {
             try!(collector.set_segment(segment_local_id, segment));
         }

src/collector/top_collector.rs

@@ -1,8 +1,8 @@
-use std::io;
 use super::Collector;
 use SegmentReader;
 use SegmentLocalId;
 use DocAddress;
+use Result;
 use std::collections::BinaryHeap;
 use std::cmp::Ordering;
 use DocId;
@@ -105,7 +105,7 @@ impl TopCollector {
 
 impl Collector for TopCollector {
 
-    fn set_segment(&mut self, segment_id: SegmentLocalId, _: &SegmentReader) -> io::Result<()> {
+    fn set_segment(&mut self, segment_id: SegmentLocalId, _: &SegmentReader) -> Result<()> {
         self.segment_id = segment_id;
         Ok(())
     }

src/common/serialize.rs

@@ -74,7 +74,6 @@ impl BinarySerializable for u64 {
 
 impl BinarySerializable for u8 {
     fn serialize(&self, writer: &mut Write) -> io::Result<usize> {
-        // TODO error
         try!(writer.write_u8(*self));
         Ok(1)
     }

src/core/index.rs

@@ -2,6 +2,7 @@ use Result;
 use Error;
 use schema::Schema;
 use std::sync::Arc;
+use std::borrow::BorrowMut;
 use std::fmt;
 use rustc_serialize::json;
 use core::SegmentId;
@@ -13,41 +14,72 @@ use num_cpus;
 use super::segment::Segment;
 use core::SegmentReader;
 use super::pool::Pool;
+use core::SegmentMeta;
 use super::pool::LeasedItem;
 use std::path::Path;
-use indexer::SegmentManager;
 use core::IndexMeta;
 use core::META_FILEPATH;
 use super::segment::create_segment;
 use indexer::segment_updater::save_new_metas;
+use directory::error::FileError;
 
 const NUM_SEARCHERS: usize = 12;
 
-/// Accessor to the index segment manager
-///
-/// This method is not part of tantivy's public API
-pub fn get_segment_manager(index: &Index) -> Arc<SegmentManager> {
-    index.segment_manager.clone()
-}
-
 
 fn load_metas(directory: &Directory) -> Result<IndexMeta> {
-    let meta_file = try!(directory.open_read(&META_FILEPATH));
-    let meta_content = String::from_utf8_lossy(meta_file.as_slice());
-    json::decode(&meta_content)
+    let meta_data = directory.atomic_read(&META_FILEPATH)?;
+    let meta_string = String::from_utf8_lossy(&meta_data);
+    json::decode(&meta_string)
         .map_err(|e| Error::CorruptedFile(META_FILEPATH.clone(), Box::new(e)))
 }
 
 /// Tantivy's Search Index
 pub struct Index {
-    segment_manager: Arc<SegmentManager>,
-
     directory: Box<Directory>,
     schema: Schema,
     searcher_pool: Arc<Pool<Searcher>>,
-    docstamp: u64,
+    opstamp: u64,
 }
 
+
+
+
+/// Deletes all of the document of the segment.
+/// This is called when there is a merge or a rollback.
+///
+/// # Disclaimer
+/// If deletion of a file fails (e.g. a file 
+/// was read-only.), the method does not
+/// fail and just logs an error when it fails.
+pub fn delete_segment(directory: &Directory, segment_id: SegmentId) {
+    info!("Deleting segment {:?}", segment_id);
+    let segment_filepaths_res = directory.ls_starting_with(
+        &*segment_id.uuid_string()
+    );
+
+    match segment_filepaths_res {
+        Ok(segment_filepaths) => {
+            for segment_filepath in &segment_filepaths {
+                if let Err(err) = directory.delete(&segment_filepath) {
+                    match err {
+                        FileError::FileDoesNotExist(_) => {
+                            // this is normal behavior.
+                            // the position file for instance may not exists.
+                        }
+                        FileError::IOError(err) => {
+                            error!("Failed to remove {:?} : {:?}", segment_id, err);
+                        }
+                    }
+                }
+            }
+        }
+        Err(_) => {
+            error!("Failed to list files of segment {:?} for deletion.", segment_id.uuid_string());
+        }
+    }
+}
+
+
 impl Index {
     /// Creates a new index using the `RAMDirectory`.
     ///
@@ -63,8 +95,7 @@ impl Index {
     ///
     /// If a previous index was in this directory, then its meta file will be destroyed.
     pub fn create(directory_path: &Path, schema: Schema) -> Result<Index> {
-        let mut directory = MmapDirectory::open(directory_path)?;
-        save_new_metas(schema.clone(), 0, &mut directory)?;
+        let directory = MmapDirectory::open(directory_path)?;
         Index::from_directory(box directory, schema)
     }
 
@@ -84,38 +115,36 @@ impl Index {
     /// Creates a new index given a directory and an `IndexMeta`.
     fn create_from_metas(directory: Box<Directory>, metas: IndexMeta) -> Result<Index> {
         let schema = metas.schema.clone();
-        let docstamp = metas.docstamp;
-        let committed_segments = metas.committed_segments;
-        // TODO log somethings is uncommitted is not empty.
+        let opstamp = metas.opstamp;
         let index = Index {
-            segment_manager: Arc::new(SegmentManager::from_segments(committed_segments)),
             directory: directory,
             schema: schema,
             searcher_pool: Arc::new(Pool::new()),
-            docstamp: docstamp,
+            opstamp: opstamp,
         };
         try!(index.load_searchers());
         Ok(index)
     }
 
-    /// Opens a new directory from a directory.
-    pub fn from_directory(directory: Box<Directory>, schema: Schema) -> Result<Index> {
+    /// Create a new index from a directory.
+    pub fn from_directory(mut directory: Box<Directory>, schema: Schema) -> Result<Index> {
+        save_new_metas(schema.clone(), 0, directory.borrow_mut())?;
         Index::create_from_metas(directory, IndexMeta::with_schema(schema))
     }
 
     /// Opens a new directory from an index path.
     pub fn open(directory_path: &Path) -> Result<Index> {
         let directory = try!(MmapDirectory::open(directory_path));
-        let metas = try!(load_metas(&directory)); //< TODO does the directory already exists?
+        let metas = try!(load_metas(&directory));
         Index::create_from_metas(directory.box_clone(), metas)
     }
 
-    /// Returns the index docstamp.
+    /// Returns the index opstamp.
     ///
-    /// The docstamp is the number of documents that have been added
+    /// The opstamp is the number of documents that have been added
     /// from the beginning of time, and until the moment of the last commit.
-    pub fn docstamp(&self) -> u64 {
-        self.docstamp
+    pub fn opstamp(&self) -> u64 {
+        self.opstamp
     }
 
     /// Creates a multithreaded writer.
@@ -151,27 +180,32 @@ impl Index {
     }
 
     /// Returns the list of segments that are searchable
-    pub fn searchable_segments(&self) -> Vec<Segment> {
-        self.searchable_segment_ids()
+    pub fn searchable_segments(&self) -> Result<Vec<Segment>> {
+        let metas = load_metas(self.directory())?; 
+        Ok(metas
+            .segments
             .into_iter()
-            .map(|segment_id| self.segment(segment_id))
-            .collect()
+            .map(|segment_meta| self.segment(segment_meta))
+            .collect())
     }
-
+    
     /// Remove all of the file associated with the segment.
     ///
     /// This method cannot fail. If a problem occurs,
     /// some files may end up never being removed.
     /// The error will only be logged.
     pub fn delete_segment(&self, segment_id: SegmentId) {
-        self.segment(segment_id).delete();
+        delete_segment(self.directory(), segment_id); 
     }
 
-    /// Return a segment object given a `segment_id`
-    ///
-    /// The segment may or may not exist.
-    pub fn segment(&self, segment_id: SegmentId) -> Segment {
-        create_segment(self.clone(), segment_id)
+    pub fn segment(&self, segment_meta: SegmentMeta) -> Segment {
+        create_segment(self.clone(), segment_meta)
+    }
+
+    /// Creates a new segment.
+    pub fn new_segment(&self) -> Segment {
+        let segment_meta = SegmentMeta::new(SegmentId::generate_random());
+        create_segment(self.clone(), segment_meta)
     }
 
     /// Return a reference to the index directory.
@@ -184,14 +218,19 @@ impl Index {
         &mut *self.directory
     }
 
-    /// Returns the list of segment ids that are searchable.
-    fn searchable_segment_ids(&self) -> Vec<SegmentId> {
-        self.segment_manager.committed_segments()
+    /// Reads the meta.json and returns the list of
+    /// segments in the last commit.
+    pub fn segments(&self) -> Result<Vec<SegmentMeta>> {
+        Ok(load_metas(self.directory())?.segments)
     }
-
-    /// Creates a new segment.
-    pub fn new_segment(&self) -> Segment {
-        self.segment(SegmentId::generate_random())
+    
+    /// Returns the list of segment ids that are searchable.
+    pub fn searchable_segment_ids(&self) -> Result<Vec<SegmentId>> {
+        Ok(load_metas(self.directory())?
+            .segments
+            .iter()
+            .map(|segment_meta| segment_meta.id())
+            .collect())           
     }
 
     /// Creates a new generation of searchers after
@@ -200,11 +239,12 @@ impl Index {
     /// This needs to be called when a new segment has been
     /// published or after a merge.
     pub fn load_searchers(&self) -> Result<()> {
-        let searchable_segments = self.searchable_segments();
+        let searchable_segments = self.searchable_segments()?;
         let mut searchers = Vec::new();
         for _ in 0..NUM_SEARCHERS {
             let searchable_segments_clone = searchable_segments.clone();
-            let segment_readers: Vec<SegmentReader> = try!(searchable_segments_clone.into_iter()
+            let segment_readers: Vec<SegmentReader> = try!(searchable_segments_clone
+                .into_iter()
                 .map(SegmentReader::open)
                 .collect());
             let searcher = Searcher::from(segment_readers);
@@ -239,12 +279,10 @@ impl fmt::Debug for Index {
 impl Clone for Index {
     fn clone(&self) -> Index {
         Index {
-            segment_manager: self.segment_manager.clone(),
-
             directory: self.directory.box_clone(),
             schema: self.schema.clone(),
             searcher_pool: self.searcher_pool.clone(),
-            docstamp: self.docstamp,
+            opstamp: self.opstamp,
         }
     }
 }

src/core/index_meta.rs

@@ -1,7 +1,5 @@
-
 use schema::Schema;
-use core::SegmentId;
-
+use core::SegmentMeta;
 
 /// Meta information about the `Index`.
 /// 
@@ -13,35 +11,17 @@ use core::SegmentId;
 ///
 #[derive(Clone,Debug,RustcDecodable,RustcEncodable)]
 pub struct IndexMeta {
-    pub committed_segments: Vec<SegmentMeta>,
-    pub uncommitted_segments: Vec<SegmentMeta>,
+    pub segments: Vec<SegmentMeta>,
     pub schema: Schema,
-    pub docstamp: u64,
+    pub opstamp: u64,
 }
 
 impl IndexMeta {
     pub fn with_schema(schema: Schema) -> IndexMeta {
         IndexMeta {
-            committed_segments: Vec::new(),
-            uncommitted_segments: Vec::new(),
+            segments: vec!(),
             schema: schema,
-            docstamp: 0u64,
+            opstamp: 0u64,
         }
     }
 }
-
-#[derive(Clone, Debug, RustcDecodable,RustcEncodable)]
-pub struct SegmentMeta {
-    pub segment_id: SegmentId,
-    pub num_docs: u32,
-}
-
-#[cfg(test)]
-impl SegmentMeta {
-    pub fn new(segment_id: SegmentId, num_docs: u32) -> SegmentMeta {
-        SegmentMeta {
-            segment_id: segment_id,
-            num_docs: num_docs,
-        }
-    }
-}

src/core/mod.rs

@@ -7,6 +7,7 @@ mod segment_component;
 mod segment;
 mod index_meta;
 mod pool;
+mod segment_meta;
 mod term_iterator;
 
 use std::path::PathBuf;
@@ -18,7 +19,8 @@ pub use self::segment::Segment;
 pub use self::segment::SegmentInfo;
 pub use self::segment::SerializableSegment;
 pub use self::index::Index;
-pub use self::index_meta::{IndexMeta, SegmentMeta};
+pub use self::segment_meta::SegmentMeta;
+pub use self::index_meta::IndexMeta;
 pub use self::term_iterator::TermIterator;
 
 lazy_static! {

src/core/searcher.rs

@@ -8,6 +8,7 @@ use DocId;
 use DocAddress;
 use schema::Term;
 use core::TermIterator;
+use std::fmt;
 
 
 /// Holds a list of `SegmentReader`s ready for search.
@@ -15,13 +16,13 @@ use core::TermIterator;
 /// It guarantees that the `Segment` will not be removed before  
 /// the destruction of the `Searcher`.
 /// 
-#[derive(Debug)]
 pub struct Searcher {
     segment_readers: Vec<SegmentReader>,
 }
 
+
 impl Searcher {
-      
+
     /// Fetches a document from tantivy's store given a `DocAddress`.
     ///
     /// The searcher uses the segment ordinal to route the
@@ -83,4 +84,14 @@ impl From<Vec<SegmentReader>> for Searcher {
             segment_readers: segment_readers,
         }
     }
+}
+
+impl fmt::Debug for Searcher {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let segment_ids = self.segment_readers
+            .iter()
+            .map(|segment_reader| segment_reader.segment_id())
+            .collect::<Vec<_>>();
+        write!(f, "Searcher({:?})", segment_ids)
+    }
 }

src/core/segment.rs

@@ -9,81 +9,72 @@ use indexer::segment_serializer::SegmentSerializer;
 use super::SegmentComponent;
 use core::Index;
 use std::result;
+use core::SegmentMeta;
 use directory::error::{FileError, OpenWriteError};
 
-
-
 /// A segment is a piece of the index.
 #[derive(Clone)]
 pub struct Segment {
     index: Index,
-    segment_id: SegmentId,
+    meta: SegmentMeta,
 }
 
 impl fmt::Debug for Segment {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "Segment({:?})", self.segment_id.uuid_string())
+        write!(f, "Segment({:?})", self.id().uuid_string())
     }
 }
 
-
 /// Creates a new segment given an `Index` and a `SegmentId`
 /// 
 /// The function is here to make it private outside `tantivy`. 
-pub fn create_segment(index: Index, segment_id: SegmentId) -> Segment {
+pub fn create_segment(index: Index, meta: SegmentMeta) -> Segment {
     Segment {
         index: index,
-        segment_id: segment_id,
+        meta: meta,
     }
 }
 
 impl Segment {
-
-
+    
     /// Returns our index's schema.
     pub fn schema(&self,) -> Schema {
         self.index.schema()
     }
+    
+    pub fn meta(&self,) -> &SegmentMeta {
+        &self.meta
+    }
+
+    pub fn meta_mut(&mut self,) -> &mut SegmentMeta {
+        &mut self.meta
+    }
 
     /// Returns the segment's id.
     pub fn id(&self,) -> SegmentId {
-        self.segment_id
+        self.meta.id()
     }
-    
 
     /// Returns the relative path of a component of our segment.
     ///  
     /// It just joins the segment id with the extension 
     /// associated to a segment component.
     pub fn relative_path(&self, component: SegmentComponent) -> PathBuf {
-        self.segment_id.relative_path(component)
+        use self::SegmentComponent::*;
+        let mut path = self.id().uuid_string();
+        path.push_str(&*match component {
+            POSITIONS => ".pos".to_string(),
+            INFO => ".info".to_string(),
+            POSTINGS => ".idx".to_string(),
+            TERMS => ".term".to_string(),
+            STORE => ".store".to_string(),
+            FASTFIELDS => ".fast".to_string(),
+            FIELDNORMS => ".fieldnorm".to_string(),
+            DELETE => {format!(".{}.del", self.meta.delete_opstamp().unwrap_or(0))},
+        });
+        PathBuf::from(path)
     }
 
-    /// Deletes all of the document of the segment.
-    /// This is called when there is a merge or a rollback.
-    ///
-    /// # Disclaimer
-    /// If deletion of a file fails (e.g. a file 
-    /// was read-only.), the method does not
-    /// fail and just logs an error
-    pub fn delete(&self,) {
-        for component in SegmentComponent::values() {
-            let rel_path = self.relative_path(component);
-            if let Err(err) = self.index.directory().delete(&rel_path) {
-                match err {
-                    FileError::FileDoesNotExist(_) => {
-                        // this is normal behavior.
-                        // the position file for instance may not exists.
-                    }
-                    FileError::IOError(err) => {
-                		error!("Failed to remove {:?} : {:?}", self.segment_id, err);
-                    }
-                }
-            }
-        }
-    }
-
-
     /// Open one of the component file for read.
     pub fn open_read(&self, component: SegmentComponent) -> result::Result<ReadOnlySource, FileError> {
         let path = self.relative_path(component);

src/core/segment_component.rs

@@ -1,5 +1,3 @@
-use std::vec::IntoIter;
-
 #[derive(Copy, Clone)]
 pub enum SegmentComponent {
     INFO,
@@ -9,32 +7,7 @@ pub enum SegmentComponent {
     FIELDNORMS,
     TERMS,
     STORE,
-}
-
-impl SegmentComponent {
-    pub fn values() -> IntoIter<SegmentComponent> {
-        vec!(
-            SegmentComponent::INFO,
-            SegmentComponent::POSTINGS,
-            SegmentComponent::POSITIONS,
-            SegmentComponent::FASTFIELDS,
-            SegmentComponent::FIELDNORMS,
-            SegmentComponent::TERMS,
-            SegmentComponent::STORE,
-        ).into_iter()
-    }
-    
-    pub fn path_suffix(&self)-> &'static str {
-        match *self {
-            SegmentComponent::POSITIONS => ".pos",
-            SegmentComponent::INFO => ".info",
-            SegmentComponent::POSTINGS => ".idx",
-            SegmentComponent::TERMS => ".term",
-            SegmentComponent::STORE => ".store",
-            SegmentComponent::FASTFIELDS => ".fast",
-            SegmentComponent::FIELDNORMS => ".fieldnorm",
-        }
-    }
+    DELETE
 }
 
 

src/core/segment_id.rs

@@ -1,11 +1,8 @@
 use uuid::Uuid;
 use std::fmt;
 use rustc_serialize::{Encoder, Decoder, Encodable, Decodable};
-use core::SegmentComponent;
-use std::path::PathBuf;
 use std::cmp::{Ordering, Ord};
 
-
 #[cfg(test)]
 use std::sync::atomic;
 
@@ -48,11 +45,6 @@ impl SegmentId {
     pub fn uuid_string(&self,) -> String {
         self.0.simple().to_string()
     }
-
-    pub fn relative_path(&self, component: SegmentComponent) -> PathBuf {
-        let filename = self.uuid_string() + component.path_suffix();
-        PathBuf::from(filename)
-    }
 }
 
 impl Encodable for SegmentId {

src/core/segment_meta.rs

@@ -0,0 +1,54 @@
+use core::SegmentId;
+
+
+#[derive(Clone, Debug, RustcDecodable,RustcEncodable)]
+struct DeleteMeta {
+    num_deleted_docs: u32,
+    opstamp: u64,
+}
+
+#[derive(Clone, Debug, RustcDecodable,RustcEncodable)]
+pub struct SegmentMeta {
+    segment_id: SegmentId,
+    num_docs: u32,
+    deletes: Option<DeleteMeta>, 
+}
+
+impl SegmentMeta {
+    pub fn new(segment_id: SegmentId) -> SegmentMeta {
+        SegmentMeta {
+            segment_id: segment_id,
+            num_docs: 0,
+            deletes: None,
+        }
+    }
+
+    pub fn id(&self) -> SegmentId {
+        self.segment_id
+    }
+
+    pub fn num_docs(&self) -> u32 {
+        self.num_docs
+    } 
+
+    pub fn delete_opstamp(&self) -> Option<u64> {
+        self.deletes
+            .as_ref()
+            .map(|delete_meta| delete_meta.opstamp)
+    }
+
+    pub fn has_deletes(&self) -> bool {
+        self.deletes.is_some()
+    }
+
+    pub fn set_num_docs(&mut self, num_docs: u32) {
+        self.num_docs = num_docs;
+    }
+
+    pub fn set_deletes(&mut self, num_deleted_docs: u32, opstamp: u64) {
+        self.deletes = Some(DeleteMeta {
+            num_deleted_docs: num_deleted_docs,
+            opstamp: opstamp,
+        });
+    }
+}

src/core/segment_reader.rs

@@ -3,6 +3,8 @@ use core::Segment;
 use core::SegmentId;
 use core::SegmentComponent;
 use schema::Term;
+use common::HasLen;
+use fastfield::delete::DeleteBitSet;
 use store::StoreReader;
 use schema::Document;
 use directory::ReadOnlySource;
@@ -44,6 +46,7 @@ pub struct SegmentReader {
     store_reader: StoreReader,
     fast_fields_reader: U32FastFieldsReader,
     fieldnorms_reader: U32FastFieldsReader,
+    delete_bitset: DeleteBitSet,
     positions_data: ReadOnlySource,
     schema: Schema,
 }
@@ -63,20 +66,27 @@ impl SegmentReader {
     /// Today, `tantivy` does not handle deletes so max doc and
     /// num_docs are the same.
     pub fn num_docs(&self) -> DocId {
-        self.segment_info.max_doc
+        self.segment_info.max_doc - self.num_deleted_docs()
     }
     
+    pub fn num_deleted_docs(&self) -> DocId {
+        self.delete_bitset.len() as DocId
+    }
+
     /// Accessor to a segment's fast field reader given a field.
-    pub fn get_fast_field_reader(&self, field: Field) -> io::Result<U32FastFieldReader> {
+    pub fn get_fast_field_reader(&self, field: Field) -> Result<U32FastFieldReader> {
         let field_entry = self.schema.get_field_entry(field);
-        match *field_entry.field_type() {
-            FieldType::Str(_) => {
-                Err(io::Error::new(io::ErrorKind::Other, "fast field are not yet supported for text fields."))
+        match field_entry.field_type() {
+            &FieldType::Str(_) => {
+                Err(Error::InvalidArgument(format!("Field <{}> is not a fast field. It is a text field, and fast text fields are not supported yet.", field_entry.name())))
             },
-            FieldType::U32(_) => {
-                // TODO check that the schema allows that
-                //Err(io::Error::new(io::ErrorKind::Other, "fast field are not yet supported for text fields."))
-                self.fast_fields_reader.get_field(field)
+            &FieldType::U32(ref u32_options) => {
+                if u32_options.is_fast() {
+                    Ok(self.fast_fields_reader.get_field(field)?)
+                }
+                else {
+                    Err(Error::InvalidArgument(format!("Field <{}> is not defined as a fast field.", field_entry.name())))
+                }
             },
         }
     }
@@ -137,6 +147,15 @@ impl SegmentReader {
             .open_read(SegmentComponent::POSITIONS)
             .unwrap_or_else(|_| ReadOnlySource::empty());
         
+        let delete_bitset =
+            if segment.meta().has_deletes() {
+                let delete_data = segment.open_read(SegmentComponent::DELETE)?;
+                DeleteBitSet::open(delete_data)
+            }
+            else {
+                DeleteBitSet::empty()
+            };
+        
         let schema = segment.schema();
         Ok(SegmentReader {
             segment_info: segment_info,
@@ -146,6 +165,7 @@ impl SegmentReader {
             store_reader: store_reader,
             fast_fields_reader: fast_fields_reader,
             fieldnorms_reader: fieldnorms_reader,
+            delete_bitset: delete_bitset,
             positions_data: positions_data,
             schema: schema,
         })
@@ -214,10 +234,15 @@ impl SegmentReader {
                 FreqHandler::new_without_freq()
             }
         };
-        Some(SegmentPostings::from_data(term_info.doc_freq, postings_data, freq_handler))
+        Some(SegmentPostings::from_data(term_info.doc_freq, postings_data, &self.delete_bitset, freq_handler))
     }
-        
+    
+
     /// Returns the posting list associated with a term.
+    ///
+    /// If the term is not found, return None.
+    /// Even when non-null, because of deletes, the posting object 
+    /// returned by this method may contain no documents.
     pub fn read_postings_all_info(&self, term: &Term) -> Option<SegmentPostings> {
         let field_entry = self.schema.get_field_entry(term.field());
         let segment_posting_option = match *field_entry.field_type() {
@@ -237,6 +262,19 @@ impl SegmentReader {
     pub fn get_term_info(&self, term: &Term) -> Option<TermInfo> {
         self.term_infos.get(term.as_slice())
     }
+
+    /// Returns the segment id
+    pub fn segment_id(&self) -> SegmentId {
+        self.segment_id
+    }
+
+    pub fn delete_bitset(&self) -> &DeleteBitSet {
+        &self.delete_bitset
+    }
+
+    pub fn is_deleted(&self, doc: DocId) -> bool {
+        self.delete_bitset.is_deleted(doc)
+    }
 }
 
 

src/core/term_iterator.rs

@@ -170,6 +170,7 @@ mod tests {
                 index_writer.commit().unwrap();
             }
         }
+        index.load_searchers().unwrap();
         let searcher = index.searcher();
         let mut term_it = searcher.terms();
         let mut terms = String::new();

src/datastruct/fstmap.rs

@@ -20,7 +20,7 @@ pub struct FstMapBuilder<W: Write, V: BinarySerializable> {
 }
 
 impl<W: Write, V: BinarySerializable> FstMapBuilder<W, V> {
-
+    
     pub fn new(w: W) -> io::Result<FstMapBuilder<W, V>> {
         let fst_builder = try!(fst::MapBuilder::new(w).map_err(convert_fst_error));
         Ok(FstMapBuilder {
@@ -30,7 +30,28 @@ impl<W: Write, V: BinarySerializable> FstMapBuilder<W, V> {
         })
     }
 
-    pub fn insert(&mut self, key: &[u8], value: &V) -> io::Result<()>{
+    /// Horribly unsafe, nobody should ever do that... except me :)
+    /// 
+    /// If used, it must be used by systematically alternating calls
+    /// to insert_key and insert_value.
+    ///
+    /// TODO see if I can bend Rust typesystem to enforce that
+    /// in a nice way.
+    pub fn insert_key(&mut self, key: &[u8]) -> io::Result<()> {
+        try!(self.fst_builder
+            .insert(key, self.data.len() as u64)
+            .map_err(convert_fst_error));
+        Ok(())
+    }
+
+    /// Horribly unsafe, nobody should ever do that... except me :)
+    pub fn insert_value(&mut self, value: &V) -> io::Result<()> {
+        try!(value.serialize(&mut self.data));
+        Ok(())
+    }
+
+    #[cfg(test)]
+    pub fn insert(&mut self, key: &[u8], value: &V) -> io::Result<()> {
         try!(self.fst_builder
             .insert(key, self.data.len() as u64)
             .map_err(convert_fst_error));
@@ -126,7 +147,6 @@ mod tests {
         assert_eq!(keys.next().unwrap(), "abc".as_bytes());
         assert_eq!(keys.next().unwrap(), "abcd".as_bytes());
         assert_eq!(keys.next(), None);
- 
     }
 
 }

src/datastruct/skip/skiplist_builder.rs

@@ -36,7 +36,7 @@ impl<T: BinarySerializable> LayerBuilder<T> {
     fn insert(&mut self, doc_id: DocId, value: &T) -> io::Result<Option<(DocId, u32)>> {
         self.remaining -= 1;
         self.len += 1;
-        let offset = self.written_size() as u32; // TODO not sure if we want after or here
+        let offset = self.written_size() as u32;
         try!(doc_id.serialize(&mut self.buffer));
         try!(value.serialize(&mut self.buffer));
         Ok(if self.remaining == 0 {

src/datastruct/stacker/hashmap.rs

@@ -125,6 +125,10 @@ impl<'a, V> HashMap<'a, V> where V: HeapAllocable {
             .map(move |addr: u32| heap.get_mut_ref::<V>(addr))
     }
 
+    pub fn heap(&self) -> &Heap {
+        &self.heap
+    }
+
     pub fn get_or_create<S: AsRef<[u8]>>(&mut self, key: S) -> &mut V {
         let entry = self.lookup(key.as_ref());
         match entry {

src/datastruct/stacker/heap.rs

@@ -41,7 +41,6 @@ impl Heap {
         self.inner().clear();
     }
     
-    
     /// Return the heap capacity.
     pub fn capacity(&self,) -> u32 {
         self.inner().capacity()
@@ -91,6 +90,10 @@ impl Heap {
     pub fn get_mut_ref<Item>(&self, addr: u32) -> &mut Item {
         self.inner().get_mut_ref(addr)
     }
+
+    pub fn get_ref<Item>(&self, addr: u32) -> &Item {
+        self.inner().get_mut_ref(addr)
+    }
 }
 
 

src/directory/directory.rs

@@ -1,13 +1,14 @@
 use std::marker::Send;
 use std::fmt;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use directory::error::{FileError, OpenWriteError};
 use directory::{ReadOnlySource, WritePtr};
 use std::result;
 use std::io;
 use std::marker::Sync;
 
-/// Write-once read many (WORM) abstraction for where tantivy's index should be stored. 
+/// Write-once read many (WORM) abstraction for where
+/// tantivy's data should be stored. 
 ///
 /// There are currently two implementations of `Directory`
 /// 
@@ -27,6 +28,8 @@ pub trait Directory: fmt::Debug + Send + Sync + 'static {
     /// have no effect on the returned `ReadOnlySource` object. 
     fn open_read(&self, path: &Path) -> result::Result<ReadOnlySource, FileError>;
     
+    fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, FileError>;
+
     /// Removes a file
     ///
     /// Removing a file will not affect an eventual
@@ -70,6 +73,10 @@ pub trait Directory: fmt::Debug + Send + Sync + 'static {
         
     /// Clones the directory and boxes the clone 
     fn box_clone(&self) -> Box<Directory>;
+
+    /// Returns the list of files starting by a given
+    /// prefix.
+    fn ls_starting_with(&self, prefix: &str) -> io::Result<Vec<PathBuf>>;
 }
 
 

src/directory/mmap_directory.rs

@@ -9,16 +9,14 @@ use fst::raw::MmapReadOnly;
 use memmap::{Mmap, Protection};
 use std::collections::hash_map::Entry as HashMapEntry;
 use std::collections::HashMap;
-use std::mem;
 use std::convert::From;
 use std::fmt;
-use std::fs;
-use std::fs::File;
+use std::fs::{self, File};
 use std::fs::OpenOptions;
-use std::io;
-use std::io::{Seek, SeekFrom};
-use std::io::BufWriter;
-use std::io::Write;
+use std::fs::ReadDir;
+use std::io::{self, Seek, SeekFrom};
+use std::io::{BufWriter, Read, Write};
+use std::mem;
 use std::path::{Path, PathBuf};
 use std::result;
 use std::sync::Arc;
@@ -302,7 +300,7 @@ impl Directory for MmapDirectory {
     }
 
     fn delete(&self, path: &Path) -> result::Result<(), FileError> {
-        debug!("Delete {:?}", path);
+        debug!("Deleting file {:?}", path);
         let full_path = self.resolve_path(path);
         let mut mmap_cache = try!(self.mmap_cache
             .write()
@@ -324,6 +322,13 @@ impl Directory for MmapDirectory {
         full_path.exists()
     }
 
+    fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, FileError> {
+        let full_path = self.resolve_path(path);
+        let mut buffer = Vec::new();
+        File::open(&full_path)?.read_to_end(&mut buffer)?;
+        Ok(buffer)
+    }
+
     fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()> {
         debug!("Atomic Write {:?}", path);
         let full_path = self.resolve_path(path);
@@ -338,6 +343,25 @@ impl Directory for MmapDirectory {
         Box::new(self.clone())
     }
     
+    fn ls_starting_with(&self, prefix: &str) -> io::Result<Vec<PathBuf>> {
+        fs::read_dir(&self.root_path)
+        .map(|paths: ReadDir| {
+            paths
+            .filter_map(|dir_entry_res|
+                dir_entry_res
+                    .ok()
+                    .map(|dir_entry| dir_entry.path())
+            )
+            .filter(|path| 
+                path.to_str()
+                    .map(|filepath| filepath.starts_with(prefix))
+                    .unwrap_or(false)
+            )
+            .map(PathBuf::from)
+            .collect()
+        })
+          
+    }
 }
 
 

src/directory/mod.rs

@@ -111,7 +111,7 @@ mod tests {
         }
     }
 
-    fn test_delete(directory: &mut Directory) {
+    fn test_directory_delete(directory: &mut Directory) {
         assert!(directory.open_read(*TEST_PATH).is_err());
         let mut write_file = directory.open_write(*TEST_PATH).unwrap();
         write_file.write_all(&[1, 2, 3, 4]).unwrap();
@@ -131,7 +131,7 @@ mod tests {
         test_seek(directory);
         test_rewrite_forbidden(directory);
         test_write_create_the_file(directory);
-        test_delete(directory);
+        test_directory_delete(directory);
     }
 
 }

src/directory/ram_directory.rs

@@ -130,6 +130,20 @@ impl InnerDirectory {
             .contains_key(path)
     }
 
+    fn ls_starting_with(&self, prefix: &str) -> Vec<PathBuf> {
+        self.0
+            .read()
+            .expect("Failed to get read lock directory.")
+            .keys()
+            .filter(|path: &&PathBuf|
+                path.to_str()
+                    .map(|p: &str| p.starts_with(prefix))
+                    .unwrap_or(false)
+            )
+            .cloned()
+            .collect()
+    }
+
 }
 
 impl fmt::Debug for RAMDirectory {
@@ -185,6 +199,12 @@ impl Directory for RAMDirectory {
         self.fs.exists(path)
     }
 
+    fn atomic_read(&self, path: &Path) -> Result<Vec<u8>, FileError> {
+        let read = self.open_read(path)?;
+        Ok(read.as_slice()
+               .to_owned())
+    }
+
     fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()> {
         let path_buf = PathBuf::from(path);
         let mut vec_writer = VecWriter::new(path_buf.clone(), self.fs.clone());
@@ -198,4 +218,9 @@ impl Directory for RAMDirectory {
         Box::new(self.clone())
     }
 
+
+    fn ls_starting_with(&self, prefix: &str) -> io::Result<Vec<PathBuf>> {
+        Ok(self.fs.ls_starting_with(prefix))
+    }
+
 }

src/error.rs

@@ -32,11 +32,11 @@ pub enum Error {
     /// The data within is corrupted.
     ///
     /// For instance, it contains invalid JSON.
-    CorruptedFile(PathBuf, Box<error::Error + Send>),
+    CorruptedFile(PathBuf, Box<error::Error + Send + Sync>),
     /// Invalid argument was passed by the user.
     InvalidArgument(String),
     /// An Error happened in one of the thread
-    ErrorInThread(String), // TODO investigate better solution
+    ErrorInThread(String),
 }
 
 impl From<io::Error> for Error {

src/fastfield/delete.rs

@@ -0,0 +1,127 @@
+use bit_set::BitSet;
+use directory::WritePtr;
+use std::io::Write;
+use std::io;
+use directory::ReadOnlySource;
+use DocId;
+use common::HasLen;
+
+pub fn write_delete_bitset(delete_bitset: &BitSet, writer: &mut WritePtr) -> io::Result<()> {
+    let max_doc = delete_bitset.capacity();
+    let mut byte = 0u8;
+    let mut shift = 0u8;
+    for doc in 0..max_doc {
+        if delete_bitset.contains(doc) {
+            byte |= 1 << shift;
+        }
+        if shift == 7 {
+            writer.write(&[byte])?;
+            shift = 0;
+            byte = 0;
+        }
+        else {
+            shift += 1;
+        }
+    }
+    if max_doc % 8 > 0 {
+        writer.write(&[byte])?;
+    }
+    writer.flush()
+}
+
+#[derive(Clone)]
+pub struct DeleteBitSet {
+    data: ReadOnlySource,
+    len: usize,  
+}
+
+impl DeleteBitSet {
+
+    pub fn open(data: ReadOnlySource) -> DeleteBitSet {
+        let num_deleted: usize = data
+            .as_slice()
+            .iter()
+            .map(|b| b.count_ones() as usize)
+            .sum();
+        DeleteBitSet {
+            data: data,
+            len: num_deleted,
+        }
+    }
+
+    pub fn empty() -> DeleteBitSet {
+        DeleteBitSet {
+            data: ReadOnlySource::empty(),
+            len: 0,
+        }
+    }
+
+    pub fn has_deletes(&self) -> bool {
+        self.len() > 0
+    }
+
+    pub fn is_deleted(&self, doc: DocId) -> bool {
+        if self.len == 0 {
+            false
+        }
+        else {
+            let byte_offset = doc / 8u32;
+            let b: u8 = (*self.data)[byte_offset as usize];
+            let shift = (doc & 7u32) as u8;
+            b & (1u8 << shift) != 0    
+        }
+    }
+
+}
+
+
+impl HasLen for DeleteBitSet {
+    fn len(&self) -> usize {
+        self.len
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::path::PathBuf;
+    use bit_set::BitSet;
+    use directory::*;
+    use super::*;
+
+    fn test_delete_bitset_helper(bitset: &BitSet) {
+        let test_path = PathBuf::from("test");
+        let mut directory = RAMDirectory::create();
+        {
+            let mut writer = directory.open_write(&*test_path).unwrap();
+            write_delete_bitset(bitset, &mut writer).unwrap();
+        }
+        {
+            let source = directory.open_read(&test_path).unwrap();
+            let delete_bitset = DeleteBitSet::open(source);
+            let n = bitset.capacity();
+            for doc in 0..n {
+                assert_eq!(bitset.contains(doc), delete_bitset.is_deleted(doc as DocId));
+            }
+            assert_eq!(delete_bitset.len(), bitset.len());
+        }
+    }
+
+    #[test]
+    fn test_delete_bitset() {
+        {
+            let mut bitset = BitSet::with_capacity(10);
+            bitset.insert(1);
+            bitset.insert(9);
+            test_delete_bitset_helper(&bitset);
+        }
+        {
+            let mut bitset = BitSet::with_capacity(8);
+            bitset.insert(1);
+            bitset.insert(2);
+            bitset.insert(3);
+            bitset.insert(5);
+            bitset.insert(7);
+            test_delete_bitset_helper(&bitset);
+        }
+    }
+}

src/fastfield/mod.rs

@@ -13,6 +13,7 @@
 mod reader;
 mod writer;
 mod serializer;
+pub mod delete;
 
 pub use self::writer::{U32FastFieldsWriter, U32FastFieldWriter};
 pub use self::reader::{U32FastFieldsReader, U32FastFieldReader};
@@ -53,9 +54,6 @@ mod tests {
     #[test]
     pub fn test_fastfield() {
         let test_fastfield = U32FastFieldReader::from(vec!(100,200,300));
-        println!("{}", test_fastfield.get(0));
-        println!("{}", test_fastfield.get(1));
-        println!("{}", test_fastfield.get(2));
         assert_eq!(test_fastfield.get(0), 100);
         assert_eq!(test_fastfield.get(1), 200);
         assert_eq!(test_fastfield.get(2), 300); 

src/fastfield/writer.rs

@@ -98,7 +98,6 @@ impl U32FastFieldWriter {
                 }
             },
             None => {
-                // TODO make default value configurable
                 0u32
             }            
         }

src/indexer/delete_queue.rs

@@ -0,0 +1,123 @@
+use super::operation::DeleteOperation;
+use std::sync::{Arc, RwLock};
+use std::mem;
+
+/// This implementation assumes that we
+/// have a lot more write operation than read operations.
+
+#[derive(Default)]
+struct InnerDeleteQueue {
+    ro_chunks: DeleteQueueSnapshot,
+    last_chunk: Vec<DeleteOperation>,
+}
+
+impl InnerDeleteQueue {
+    pub fn push(&mut self, delete_operation: DeleteOperation) {
+        self.last_chunk.push(delete_operation);
+    }
+
+    pub fn snapshot(&mut self,) -> DeleteQueueSnapshot {
+        if self.last_chunk.len() > 0 {
+            let new_operations = vec!();
+            let new_ro_chunk = mem::replace(&mut self.last_chunk, new_operations);
+            self.ro_chunks.push(new_ro_chunk)
+        }
+        self.ro_chunks.clone()
+    }
+
+    pub fn clear(&mut self) {
+        self.ro_chunks.clear();
+        self.last_chunk.clear();
+    }
+}
+
+
+
+#[derive(Default, Clone)]
+pub struct DeleteQueueSnapshot(Vec<Arc<Vec<DeleteOperation>>>);
+
+impl DeleteQueueSnapshot {
+    fn push(&mut self, operations: Vec<DeleteOperation>) {
+        self.0.push(Arc::new(operations));
+    }
+
+    pub fn iter<'a>(&'a self) -> impl Iterator<Item=&'a DeleteOperation> {
+        self.0
+            .iter()
+            .flat_map(|chunk| chunk.iter())
+    }
+
+    pub fn clear(&mut self) {
+        self.0.clear();
+    }
+}
+
+#[derive(Clone, Default)]
+pub struct DeleteQueue(Arc<RwLock<InnerDeleteQueue>>);
+
+impl DeleteQueue {
+    pub fn push(&self, delete_operation: DeleteOperation) {
+        self.0.write().unwrap().push(delete_operation);
+    }
+
+    pub fn snapshot(&self) -> DeleteQueueSnapshot {
+        self.0.write().unwrap().snapshot()
+    }
+
+    pub fn clear(&self) {
+        self.0.write().unwrap().clear();
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::{DeleteQueue, DeleteOperation};
+    use schema::{Term, Field};
+
+    #[test]
+    fn test_deletequeue() {
+        let delete_queue = DeleteQueue::default();
+        
+        let make_op = |i: usize| {
+            let field = Field(1u8);
+            DeleteOperation {
+                opstamp: i as u64,
+                term: Term::from_field_u32(field, i as u32)
+            }
+        };
+
+        delete_queue.push(make_op(1));
+        delete_queue.push(make_op(2));
+
+        let snapshot = delete_queue.snapshot();
+        {
+            let mut operations_it = snapshot.iter();
+            assert_eq!(operations_it.next().unwrap().opstamp, 1);
+            assert_eq!(operations_it.next().unwrap().opstamp, 2);
+            assert!(operations_it.next().is_none());
+        }
+        {   // iterating does not consume results.
+            let mut operations_it = snapshot.iter();
+            assert_eq!(operations_it.next().unwrap().opstamp, 1);
+            assert_eq!(operations_it.next().unwrap().opstamp, 2);
+            assert!(operations_it.next().is_none());
+        }
+        // operations does not own a lock on the queue.
+        delete_queue.push(make_op(3));
+        let snapshot2 = delete_queue.snapshot();
+        {
+            // operations is not affected by
+            // the push that occurs after.
+            let mut operations_it = snapshot.iter();
+            let mut operations2_it = snapshot2.iter();
+            assert_eq!(operations_it.next().unwrap().opstamp, 1);
+            assert_eq!(operations2_it.next().unwrap().opstamp, 1);
+            assert_eq!(operations_it.next().unwrap().opstamp, 2);
+            assert_eq!(operations2_it.next().unwrap().opstamp, 2);
+            assert!(operations_it.next().is_none());
+            assert_eq!(operations2_it.next().unwrap().opstamp, 3);
+            assert!(operations2_it.next().is_none());
+        }
+    }
+}

src/indexer/doc_opstamp_mapping.rs

@@ -0,0 +1,93 @@
+use std::sync::Arc;
+use DocId;
+
+
+// Doc to opstamp is used to identify which
+// document should be deleted.
+// 
+// Since the docset matching the query of a delete operation
+// is not computed right when the delete operation is received,
+// we need to find a way to evaluate, for each document,
+// whether the document was added before or after
+// the delete operation. This anteriority is used by comparing
+// the docstamp of the document.
+//
+// The doc to opstamp mapping stores precisely an array
+// indexed by doc id and storing the opstamp of the document.
+// 
+// This mapping is (for the moment) stricly increasing
+// because of the way document id are allocated.
+#[derive(Clone)]
+pub enum DocToOpstampMapping {
+    WithMap(Arc<Vec<u64>>),
+    None
+}
+
+impl From<Vec<u64>> for DocToOpstampMapping {
+    fn from(opstamps: Vec<u64>) -> DocToOpstampMapping {
+        DocToOpstampMapping::WithMap(Arc::new(opstamps))
+    }
+}
+
+impl DocToOpstampMapping {
+    
+    /// Given an opstamp return the limit doc id L
+    /// such that all doc id D such that 
+    // D >= L iff opstamp(D) >= than `target_opstamp`.
+    //
+    // The edge case opstamp = some doc opstamp is in practise
+    // never called.
+    pub fn compute_doc_limit(&self, target_opstamp: u64) -> DocId {
+        match *self {
+            DocToOpstampMapping::WithMap(ref doc_opstamps) => {
+                match doc_opstamps.binary_search(&target_opstamp) {
+                    Ok(doc_id) => doc_id as DocId,
+                    Err(doc_id) => doc_id as DocId,
+                }
+            }
+            DocToOpstampMapping::None => DocId::max_value(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::DocToOpstampMapping;
+
+    #[test]
+    fn test_doc_to_opstamp_mapping_none() {
+        let doc_to_opstamp_mapping = DocToOpstampMapping::None;
+        assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(1), u32::max_value());
+    }
+
+    #[test]
+    fn test_doc_to_opstamp_mapping_complex() {
+        {
+            let doc_to_opstamp_mapping = DocToOpstampMapping::from(vec!());
+            assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(0u64), 0);
+            assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(2u64), 0);
+        }
+        {
+            let doc_to_opstamp_mapping = DocToOpstampMapping::from(vec!(1u64));
+            assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(0u64), 0);
+            assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(2u64), 1);
+        }
+        {
+            let doc_to_opstamp_mapping = DocToOpstampMapping::from(vec!(1u64, 12u64, 17u64, 23u64));
+            assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(0u64), 0);
+            for i in 2u64..13u64 {
+                assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(i), 1);
+            }
+            for i in 13u64..18u64 {
+                assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(i), 2);
+            }
+            for i in 18u64..24u64 {
+                assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(i), 3);
+            }
+            for i in 24u64..30u64 {
+                assert_eq!(doc_to_opstamp_mapping.compute_doc_limit(i), 4);
+            }
+        }
+    }
+}

src/indexer/index_writer.rs

@@ -1,30 +1,37 @@
-use schema::Schema;
-use schema::Document;
-use indexer::SegmentSerializer;
-use core::SerializableSegment;
+use bit_set::BitSet;
+use chan;
 use core::Index;
 use core::Segment;
-use std::thread::JoinHandle;
-use indexer::{MergePolicy, DefaultMergePolicy};
-use indexer::SegmentWriter;
-use super::directory_lock::DirectoryLock;
-use std::clone::Clone;
-use std::io;
-use std::thread;
-use std::mem;
-use indexer::merger::IndexMerger;
+use core::SegmentComponent;
 use core::SegmentId;
-use datastruct::stacker::Heap;
-use std::mem::swap;
-use std::sync::{Arc, Mutex};
-use chan;
 use core::SegmentMeta;
-use super::segment_updater::{SegmentUpdater, SegmentUpdate, SegmentUpdateSender};
-use std::time::Duration;
-use super::super::core::index::get_segment_manager;
-use super::segment_manager::CommitState;
-use Result;
+use core::SegmentReader;
+use datastruct::stacker::Heap;
 use Error;
+use fastfield::delete::write_delete_bitset;
+use indexer::delete_queue::DeleteQueueSnapshot;
+use futures::Canceled;
+use futures::Future;
+use indexer::delete_queue::DeleteQueue;
+use indexer::doc_opstamp_mapping::DocToOpstampMapping;
+use indexer::MergePolicy;
+use indexer::operation::DeleteOperation;
+use indexer::SegmentEntry;
+use indexer::SegmentWriter;
+use postings::DocSet;
+use postings::SegmentPostingsOption;
+use Result;
+use schema::Document;
+use schema::Schema;
+use schema::Term;
+use std::io;
+use std::mem;
+use std::mem::swap; 
+use std::thread;
+use std::thread::JoinHandle;
+use super::directory_lock::DirectoryLock;
+use super::operation::AddOperation;
+use super::segment_updater::SegmentUpdater;
 
 // Size of the margin for the heap. A segment is closed when the remaining memory
 // in the heap goes below MARGIN_IN_BYTES.
@@ -36,12 +43,8 @@ pub const HEAP_SIZE_LIMIT: u32 = MARGIN_IN_BYTES * 3u32;
 // Add document will block if the number of docs waiting in the queue to be indexed reaches PIPELINE_MAX_SIZE_IN_DOCS
 const PIPELINE_MAX_SIZE_IN_DOCS: usize = 10_000;
 
-
-
-type DocumentSender = chan::Sender<Document>;
-type DocumentReceiver = chan::Receiver<Document>;
-
-
+type DocumentSender = chan::Sender<AddOperation>;
+type DocumentReceiver = chan::Receiver<AddOperation>;
 
 /// `IndexWriter` is the user entry-point to add document to an index.
 ///
@@ -55,9 +58,8 @@ pub struct IndexWriter {
     // lifetime of the lock with that of the IndexWriter.
     _directory_lock: DirectoryLock, 
     
-    _merge_policy: Arc<Mutex<Box<MergePolicy>>>,
-    
     index: Index,
+
     heap_size_in_bytes_per_thread: usize,
 
     workers_join_handle: Vec<JoinHandle<Result<()>>>,
@@ -65,15 +67,18 @@ pub struct IndexWriter {
     document_receiver: DocumentReceiver,
     document_sender: DocumentSender,
 
-    segment_update_sender: SegmentUpdateSender,
-    segment_update_thread: JoinHandle<()>,
+    segment_updater: SegmentUpdater,
 
     worker_id: usize,
 
     num_threads: usize,
 
-    uncommitted_docstamp: u64,
-    committed_docstamp: u64,
+    generation: usize,
+
+    delete_queue: DeleteQueue,
+
+    uncommitted_opstamp: u64,
+    committed_opstamp: u64,
 }
 
 // IndexWriter cannot be sent to another thread.
@@ -81,12 +86,60 @@ impl !Send for IndexWriter {}
 impl !Sync for IndexWriter {}
 
 
+// TODO put delete bitset in segment entry
+// rather than DocToOpstamp.
+
+// TODO skip delete operation before teh 
+// last delete opstamp
+
+pub fn advance_deletes(
+    segment: &mut Segment,
+    delete_operations: &DeleteQueueSnapshot,
+    doc_opstamps: &DocToOpstampMapping) -> Result<SegmentMeta> {
+        let segment_reader = SegmentReader::open(segment.clone())?;
+        let mut delete_bitset = BitSet::with_capacity(segment_reader.max_doc() as usize);
+        
+        let mut last_opstamp_opt: Option<u64> = None;
+
+        for delete_op in delete_operations.iter() {
+            // A delete operation should only affect
+            // document that were inserted after it.
+            // 
+            // Limit doc helps identify the first document
+            // that may be affected by the delete operation.
+            let limit_doc = doc_opstamps.compute_doc_limit(delete_op.opstamp);
+            if let Some(mut docset) = segment_reader.read_postings(&delete_op.term, SegmentPostingsOption::NoFreq) {
+                while docset.advance() {
+                    let deleted_doc = docset.doc();
+                    if deleted_doc < limit_doc {
+                        delete_bitset.insert(deleted_doc as usize);
+                    }
+                }
+                last_opstamp_opt = Some(delete_op.opstamp);
+            }
+        }
+
+        if let Some(last_opstamp) = last_opstamp_opt {
+            for doc in 0u32..segment_reader.max_doc() {
+                if segment_reader.is_deleted(doc) {
+                    delete_bitset.insert(doc as usize);
+                }
+            }
+            let num_deleted_docs = delete_bitset.len();
+            segment.meta_mut().set_deletes(num_deleted_docs as u32, last_opstamp);
+            let mut delete_file = segment.open_write(SegmentComponent::DELETE)?;    
+            write_delete_bitset(&delete_bitset, &mut delete_file)?;
+        }
+        Ok(segment.meta().clone())
+}
+
 fn index_documents(heap: &mut Heap,
                    segment: Segment,
                    schema: &Schema,
-                   document_iterator: &mut Iterator<Item = Document>,
-                   segment_update_sender: &mut SegmentUpdateSender)
-                   -> Result<()> {
+                   generation: usize,
+                   document_iterator: &mut Iterator<Item=AddOperation>,
+                   segment_updater: &mut SegmentUpdater)
+                   -> Result<bool> {
     heap.clear();
     let segment_id = segment.id();
     let mut segment_writer = try!(SegmentWriter::for_segment(heap, segment, &schema));
@@ -99,14 +152,24 @@ fn index_documents(heap: &mut Heap,
         }
     }
     let num_docs = segment_writer.max_doc();
-    let segment_meta = SegmentMeta {
-        segment_id: segment_id,
-        num_docs: num_docs,
-    };
+    
+    // this is ensured by the call to peek before starting
+    // the worker thread.
+    assert!(num_docs > 0);    
 
-    try!(segment_writer.finalize());
-    segment_update_sender.send(SegmentUpdate::AddSegment(segment_meta));
-    Ok(())
+    let doc_opstamps: Vec<u64> = segment_writer.finalize()?;
+
+    // let segment_entry = advance_deletes(&mut segment, delete_queue, delete_position, )?;
+    let mut segment_meta = SegmentMeta::new(segment_id);
+    segment_meta.set_num_docs(num_docs);
+
+    let mut segment_entry = SegmentEntry::new(segment_meta);
+    segment_entry.set_doc_to_opstamp(DocToOpstampMapping::from(doc_opstamps));
+
+    segment_updater
+        .add_segment(generation, segment_entry)
+        .wait()
+        .map_err(|_| Error::ErrorInThread("Could not add segment.".to_string()))
 
 }
 
@@ -114,11 +177,9 @@ fn index_documents(heap: &mut Heap,
 impl IndexWriter {
     /// The index writer
     pub fn wait_merging_threads(mut self) -> Result<()> {
-
-        self.segment_update_sender.send(SegmentUpdate::Terminate);
         
         // this will stop the indexing thread,
-        // dropping the last reference to the segment_update_sender.
+        // dropping the last reference to the segment_updater.
         drop(self.document_sender);
         
         let mut v = Vec::new();
@@ -131,12 +192,12 @@ impl IndexWriter {
                 }));
         }
         drop(self.workers_join_handle);
-        self.segment_update_thread
-            .join()
-            .map_err(|err| {
-                error!("Error in the merging thread {:?}", err);
-                Error::ErrorInThread(format!("{:?}", err))
-            })
+
+        self.segment_updater
+            .wait_merging_thread()
+            .map_err(|_| 
+                Error::ErrorInThread("Failed to join merging thread.".to_string())
+            )
     }
 
     /// Spawns a new worker thread for indexing.
@@ -146,33 +207,47 @@ impl IndexWriter {
         let index = self.index.clone();
         let schema = self.index.schema();
         let document_receiver_clone = self.document_receiver.clone();
-        let mut segment_update_sender = self.segment_update_sender.clone();
+        let mut segment_updater = self.segment_updater.clone();
         let mut heap = Heap::with_capacity(self.heap_size_in_bytes_per_thread);
-        let join_handle: JoinHandle<Result<()>> = try!(thread::Builder::new()
-            .name(format!("indexing_thread_{}", self.worker_id))
+        
+        let generation = self.generation;
+
+        let join_handle: JoinHandle<Result<()>> =
+            thread::Builder::new()
+            .name(format!("indexing thread {} for gen {}", self.worker_id, generation))
             .spawn(move || {
+                
                 loop {
-                    let segment = index.new_segment();
                     let mut document_iterator = document_receiver_clone.clone()
                         .into_iter()
                         .peekable();
+
                     // the peeking here is to avoid
                     // creating a new segment's files
                     // if no document are available.
+                    //
+                    // this is a valid guarantee as the 
+                    // peeked document now belongs to
+                    // our local iterator.
                     if document_iterator.peek().is_some() {
-                        try!(index_documents(&mut heap,
-                                             segment,
-                                             &schema,
-                                             &mut document_iterator,
-                                             &mut segment_update_sender));
-                    } else {
+                        let segment = index.new_segment();
+                        index_documents(&mut heap,
+                                        segment,
+                                        &schema,
+                                        generation,
+                                        &mut document_iterator,
+                                        &mut segment_updater)?;
+                    }
+                    else {
                         // No more documents.
                         // Happens when there is a commit, or if the `IndexWriter`
                         // was dropped.
-                        return Ok(());
+                        return Ok(())
                     }
+
+                    
                 }
-            }));
+            })?;
         self.worker_id += 1;
         self.workers_join_handle.push(join_handle);
         Ok(())
@@ -207,46 +282,48 @@ impl IndexWriter {
         
         let (document_sender, document_receiver): (DocumentSender, DocumentReceiver) =
             chan::sync(PIPELINE_MAX_SIZE_IN_DOCS);
+
+
+        let delete_queue = DeleteQueue::default();
         
-        let merge_policy: Arc<Mutex<Box<MergePolicy>>> = Arc::new(Mutex::new(box DefaultMergePolicy::default()));
-        
-        let (segment_update_sender, segment_update_thread) = SegmentUpdater::start_updater(index.clone(), merge_policy.clone());
+        let segment_updater = SegmentUpdater::new(index.clone(), delete_queue.clone())?;
         
         let mut index_writer = IndexWriter {
             
             _directory_lock: directory_lock,
             
-            _merge_policy: merge_policy,
-            
             heap_size_in_bytes_per_thread: heap_size_in_bytes_per_thread,
             index: index.clone(),
 
             document_receiver: document_receiver,
             document_sender: document_sender,
 
-            segment_update_sender: segment_update_sender,
-            segment_update_thread: segment_update_thread,
+            segment_updater: segment_updater,
 
             workers_join_handle: Vec::new(),
             num_threads: num_threads,
 
-            committed_docstamp: index.docstamp(),
-            uncommitted_docstamp: index.docstamp(),
+            delete_queue: delete_queue,
+
+            committed_opstamp: index.opstamp(),
+            uncommitted_opstamp: index.opstamp(),
+
+            generation: 0,
+
             worker_id: 0,
         };
         try!(index_writer.start_workers());
         Ok(index_writer)
     }
     
-    
-    /// Returns a clone of the index_writer merge policy.
+
     pub fn get_merge_policy(&self) -> Box<MergePolicy> {
-        self._merge_policy.lock().unwrap().box_clone() 
+        self.segment_updater.get_merge_policy()
     }
-    
+
     /// Set the merge policy.
     pub fn set_merge_policy(&self, merge_policy: Box<MergePolicy>) {
-        *self._merge_policy.lock().unwrap() = merge_policy;
+        self.segment_updater.set_merge_policy(merge_policy);
     }
     
     fn start_workers(&mut self) -> Result<()> {
@@ -257,55 +334,8 @@ impl IndexWriter {
     }
 
     /// Merges a given list of segments
-    pub fn merge(&mut self, segments: &[Segment]) -> Result<()> {
-
-        if segments.len() < 2 {
-            // no segments or one segment? nothing to do.
-            return Ok(());
-        }
-
-
-        let segment_manager = get_segment_manager(&self.index);
-
-        {
-            // let's check that all these segments are in the same
-            // committed/uncommited state.
-            let first_commit_state = segment_manager.is_committed(segments[0].id());
-
-            for segment in segments {
-                let commit_state = segment_manager.is_committed(segment.id());
-                if commit_state == CommitState::Missing {
-                    return Err(Error::InvalidArgument(format!("Segment {:?} is not in the index",
-                                                              segments[0].id())));
-                }
-                if commit_state != first_commit_state {
-                    return Err(Error::InvalidArgument(String::from("You may not merge segments \
-                                                                    that are heterogenously in \
-                                                                    committed and uncommited.")));
-                }
-            }
-        }
-
-        let schema = self.index.schema();
-
-        // An IndexMerger is like a "view" of our merged segments.
-        let merger = try!(IndexMerger::open(schema, segments));
-        let mut merged_segment = self.index.new_segment();
-
-        // ... we just serialize this index merger in our new segment
-        // to merge the two segments.
-        let segment_serializer = try!(SegmentSerializer::for_segment(&mut merged_segment));
-        let num_docs = try!(merger.write(segment_serializer));
-        let merged_segment_ids: Vec<SegmentId> =
-            segments.iter().map(|segment| segment.id()).collect();
-        let segment_meta = SegmentMeta {
-            segment_id: merged_segment.id(),
-            num_docs: num_docs,
-        };
-
-        segment_manager.end_merge(&merged_segment_ids, &segment_meta);
-        try!(self.index.load_searchers());
-        Ok(())
+    pub fn merge(&mut self, segment_ids: &[SegmentId]) -> impl Future<Item=SegmentEntry, Error=Canceled> {
+        self.segment_updater.start_merge(segment_ids)
     }
 
     /// Closes the current document channel send.
@@ -331,11 +361,15 @@ impl IndexWriter {
     /// After calling rollback, the index is in the same
     /// state as it was after the last commit.
     ///
-    /// The docstamp at the last commit is returned.
+    /// The opstamp at the last commit is returned.
     pub fn rollback(&mut self) -> Result<u64> {
 
-        self.segment_update_sender.send(SegmentUpdate::CancelGeneration);
+        // by updating the generation in the segment updater,
+        // pending add segment commands will be dismissed.
+        self.generation += 1;
 
+        let rollback_future = self.segment_updater.rollback(self.generation);
+        
         // we cannot drop segment ready receiver yet
         // as it would block the workers.
         let document_receiver = self.recreate_document_channel();
@@ -347,7 +381,7 @@ impl IndexWriter {
         let mut former_workers_join_handle = Vec::new();
         swap(&mut former_workers_join_handle,
              &mut self.workers_join_handle);
-
+        
         // wait for all the worker to finish their work
         // (it should be fast since we consumed all pending documents)
         for worker_handle in former_workers_join_handle {
@@ -361,27 +395,21 @@ impl IndexWriter {
 
         // All of our indexing workers for the rollbacked generation have
         // been terminated.
+        // 
         // Our document receiver pipe was drained.
         // No new document have been added in the meanwhile because `IndexWriter`
         // is not shared by different threads.
-        //
-        // We can now open a new generation and reaccept segments
-        // from now on.
-        self.segment_update_sender.send(SegmentUpdate::NewGeneration);
-
-        let rollbacked_segments = get_segment_manager(&self.index).rollback();
-        for segment_id in rollbacked_segments {
-
-            // TODO all delete must happen after saving
-            // meta.json
-            self.index.delete_segment(segment_id);
-        }
         
-        // reset the docstamp
-        self.uncommitted_docstamp = self.committed_docstamp;
-        Ok(self.committed_docstamp)
-    }
+        rollback_future.wait().map_err(|_|
+            Error::ErrorInThread("Error while waiting for rollback.".to_string())
+        )?;
 
+        self.delete_queue.clear();
+
+        // reset the opstamp
+        self.uncommitted_opstamp = self.committed_opstamp;
+        Ok(self.committed_opstamp)
+    }
 
     /// Commits all of the pending changes
     ///
@@ -394,7 +422,7 @@ impl IndexWriter {
     /// long as the hard disk is spared), it will be possible
     /// to resume indexing from this point.
     ///
-    /// Commit returns the `docstamp` of the last document
+    /// Commit returns the `opstamp` of the last document
     /// that made it in the commit.
     ///
     pub fn commit(&mut self) -> Result<u64> {
@@ -403,13 +431,10 @@ impl IndexWriter {
         // and recreate a new one channels.
         self.recreate_document_channel();
 
-        // Docstamp of the last document in this commit.
-        self.committed_docstamp = self.uncommitted_docstamp;
-
         let mut former_workers_join_handle = Vec::new();
         swap(&mut former_workers_join_handle,
              &mut self.workers_join_handle);
-
+        
         for worker_handle in former_workers_join_handle {
             let indexing_worker_result = try!(worker_handle.join()
                 .map_err(|e| Error::ErrorInThread(format!("{:?}", e))));
@@ -417,6 +442,7 @@ impl IndexWriter {
             // add a new worker for the next generation.
             try!(self.add_indexing_worker());
         }
+
         // here, because we join all of the worker threads,
         // all of the segment update for this commit have been
         // sent.
@@ -427,44 +453,65 @@ impl IndexWriter {
 
         // This will move uncommitted segments to the state of
         // committed segments.
-        self.segment_update_sender.send(SegmentUpdate::Commit(self.committed_docstamp));
+        self.committed_opstamp = self.stamp();
 
         // wait for the segment update thread to have processed the info
-        let segment_manager = get_segment_manager(&self.index);
-        while segment_manager.docstamp() != self.committed_docstamp {
-            thread::sleep(Duration::from_millis(100));
-        }
+        self.segment_updater
+            .commit(self.committed_opstamp)
+            .wait()?;
+        
+        self.delete_queue.clear();
+        Ok(self.committed_opstamp)
+    }    
 
-        Ok(self.committed_docstamp)
+    
+    pub fn delete_term(&mut self, term: Term) {
+        let opstamp = self.stamp();
+        let delete_operation = DeleteOperation {
+            opstamp: opstamp,
+            term: term,
+        };
+        self.delete_queue.push(delete_operation);
     }
 
+    fn stamp(&mut self) -> u64 {
+        let opstamp = self.uncommitted_opstamp;
+        self.uncommitted_opstamp += 1u64;
+        opstamp
+    }
 
     /// Adds a document.
     ///
     /// If the indexing pipeline is full, this call may block.
     ///
-    /// The docstamp is an increasing `u64` that can
+    /// The opstamp is an increasing `u64` that can
     /// be used by the client to align commits with its own
     /// document queue.
     ///
     /// Currently it represents the number of documents that
     /// have been added since the creation of the index.
-    pub fn add_document(&mut self, doc: Document) -> io::Result<u64> {
-        self.document_sender.send(doc);
-        self.uncommitted_docstamp += 1;
-        Ok(self.uncommitted_docstamp)
+    pub fn add_document(&mut self, document: Document) -> io::Result<u64> {
+        let opstamp = self.stamp();
+        let add_operation = AddOperation {
+            opstamp: opstamp,
+            document: document,
+        };
+        self.document_sender.send(add_operation);
+        Ok(opstamp)
     }
 }
 
+
+
+
 #[cfg(test)]
 mod tests {
 
-
+    use indexer::NoMergePolicy;
     use schema::{self, Document};
     use Index;
     use Term;
     use Error;
-    use indexer::NoMergePolicy;
 
     #[test]
     fn test_lockfile_stops_duplicates() {
@@ -506,7 +553,6 @@ mod tests {
         let text_field = schema_builder.add_text_field("text", schema::TEXT);
         let index = Index::create_in_ram(schema_builder.build());
 
-
         let num_docs_containing = |s: &str| {
             let searcher = index.searcher();
             let term_a = Term::from_field_text(text_field, s);
@@ -535,11 +581,12 @@ mod tests {
                 index_writer.add_document(doc).unwrap();
             }
             assert_eq!(index_writer.commit().unwrap(), 2u64);
-
+            index.load_searchers().unwrap();
             assert_eq!(num_docs_containing("a"), 0);
             assert_eq!(num_docs_containing("b"), 1);
             assert_eq!(num_docs_containing("c"), 1);
         }
+        index.load_searchers().unwrap();
         index.searcher();
     }
 
@@ -571,8 +618,9 @@ mod tests {
             // this should create 8 segments and trigger a merge.
             index_writer.commit().expect("commit failed");
             index_writer.wait_merging_threads().expect("waiting merging thread failed");
+            index.load_searchers().unwrap();
             assert_eq!(num_docs_containing("a"), 200);
-            assert_eq!(index.searchable_segments().len(), 1);
+            assert_eq!(index.searchable_segments().unwrap().len(), 1);
         }
     }
 

src/indexer/log_merge_policy.rs

@@ -48,13 +48,12 @@ impl LogMergePolicy {
 
 impl MergePolicy for LogMergePolicy {
     fn compute_merge_candidates(&self, segments: &[SegmentMeta]) -> Vec<MergeCandidate> {
-
         if segments.is_empty() {
             return Vec::new();
         }
 
         let mut size_sorted_tuples = segments.iter()
-            .map(|x| x.num_docs)
+            .map(|x| x.num_docs())
             .enumerate()
             .collect::<Vec<(usize, u32)>>();
 
@@ -75,16 +74,15 @@ impl MergePolicy for LogMergePolicy {
             levels.last_mut().unwrap().push(ind);
         }
 
-        let result = levels.iter()
+        levels
+            .iter()
             .filter(|level| level.len() >= self.min_merge_size)
             .map(|ind_vec| {
                 MergeCandidate(ind_vec.iter()
-                    .map(|&ind| segments[ind].segment_id)
+                    .map(|&ind| segments[ind].id())
                     .collect())
             })
-            .collect();
-
-        result
+            .collect()
     }
     
     fn box_clone(&self) -> Box<MergePolicy> {
@@ -122,11 +120,17 @@ mod tests {
         assert!(result_list.is_empty());
     }
 
+    fn seg_meta(num_docs: u32) -> SegmentMeta {
+        let mut segment_metas = SegmentMeta::new(SegmentId::generate_random());
+        segment_metas.set_num_docs(num_docs);
+        segment_metas
+    }
+
     #[test]
     fn test_log_merge_policy_pair() {
-        let test_input = vec![SegmentMeta::new(SegmentId::generate_random(), 10),
-                              SegmentMeta::new(SegmentId::generate_random(), 10),
-                              SegmentMeta::new(SegmentId::generate_random(), 10)];
+        let test_input = vec![seg_meta(10),
+                              seg_meta(10),
+                              seg_meta(10)];
         let result_list = test_merge_policy().compute_merge_candidates(&test_input);
         assert_eq!(result_list.len(), 1);
     }
@@ -134,12 +138,12 @@ mod tests {
     #[test]
     fn test_log_merge_policy_levels() {
         // multiple levels all get merged correctly
-        let test_input = vec![SegmentMeta::new(SegmentId::generate_random(), 10),
-                              SegmentMeta::new(SegmentId::generate_random(), 10),
-                              SegmentMeta::new(SegmentId::generate_random(), 10),
-                              SegmentMeta::new(SegmentId::generate_random(), 1000),
-                              SegmentMeta::new(SegmentId::generate_random(), 1000),
-                              SegmentMeta::new(SegmentId::generate_random(), 1000)];
+        let test_input = vec![seg_meta(10),
+                              seg_meta(10),
+                              seg_meta(10),
+                              seg_meta(1000),
+                              seg_meta(1000),
+                              seg_meta(1000)];
         let result_list = test_merge_policy().compute_merge_candidates(&test_input);
         assert_eq!(result_list.len(), 2);
     }
@@ -147,24 +151,24 @@ mod tests {
     #[test]
     fn test_log_merge_policy_within_levels() {
         // multiple levels all get merged correctly
-        let test_input = vec![SegmentMeta::new(SegmentId::generate_random(), 10),
-                              SegmentMeta::new(SegmentId::generate_random(), 11),
-                              SegmentMeta::new(SegmentId::generate_random(), 12),
-                              SegmentMeta::new(SegmentId::generate_random(), 1000),
-                              SegmentMeta::new(SegmentId::generate_random(), 1000),
-                              SegmentMeta::new(SegmentId::generate_random(), 1000)];
+        let test_input = vec![seg_meta(10),
+                              seg_meta(11),
+                              seg_meta(12),
+                              seg_meta(1000),
+                              seg_meta(1000),
+                              seg_meta(1000)];
         let result_list = test_merge_policy().compute_merge_candidates(&test_input);
         assert_eq!(result_list.len(), 2);
     }
     #[test]
     fn test_log_merge_policy_small_segments() {
         // multiple levels all get merged correctly
-        let test_input = vec![SegmentMeta::new(SegmentId::generate_random(), 1),
-                              SegmentMeta::new(SegmentId::generate_random(), 1),
-                              SegmentMeta::new(SegmentId::generate_random(), 1),
-                              SegmentMeta::new(SegmentId::generate_random(), 2),
-                              SegmentMeta::new(SegmentId::generate_random(), 2),
-                              SegmentMeta::new(SegmentId::generate_random(), 2)];
+        let test_input = vec![seg_meta(1),
+                              seg_meta(1),
+                              seg_meta(1),
+                              seg_meta(2),
+                              seg_meta(2),
+                              seg_meta(2)];
         let result_list = test_merge_policy().compute_merge_candidates(&test_input);
         assert_eq!(result_list.len(), 1);
     }

src/indexer/merge_policy.rs

@@ -13,7 +13,7 @@ pub struct MergeCandidate(pub Vec<SegmentId>);
 /// 
 /// Every time a the list of segments changes, the segment updater
 /// asks the merge policy if some segments should be merged.
-pub trait MergePolicy: marker::Send + Debug {
+pub trait MergePolicy: marker::Send + marker::Sync + Debug {
     /// Given the list of segment metas, returns the list of merge candidates. 
     ///
     /// This call happens on the segment updater thread, and will block 

src/indexer/merger.rs

@@ -3,21 +3,23 @@ use core::SegmentReader;
 use core::Segment;
 use DocId;
 use core::SerializableSegment;
+use schema::FieldValue;
 use indexer::SegmentSerializer;
 use postings::PostingsSerializer;
+use fastfield::U32FastFieldReader;
+use itertools::Itertools;
 use postings::Postings;
 use postings::DocSet;
 use core::TermIterator;
+use fastfield::delete::DeleteBitSet;
 use schema::{Schema, Field};
 use fastfield::FastFieldSerializer;
 use store::StoreWriter;
-use postings::ChainedPostings;
-use postings::HasLen;
-use postings::OffsetPostings;
 use core::SegmentInfo;
 use std::cmp::{min, max};
 use std::iter;
 
+
 pub struct IndexMerger {
     schema: Schema,
     readers: Vec<SegmentReader>,
@@ -47,14 +49,45 @@ impl DeltaPositionComputer {
     }
 }
 
+
+fn compute_min_max_val(u32_reader: &U32FastFieldReader, max_doc: DocId, delete_bitset: &DeleteBitSet) -> Option<(u32, u32)> {
+    if max_doc == 0 {
+        None
+    }
+    else if !delete_bitset.has_deletes() {
+        // no deleted documents, 
+        // we can use the previous min_val, max_val.
+        Some((u32_reader.min_val(), u32_reader.max_val()))
+    }
+    else {
+        // some deleted documents,
+        // we need to recompute the max / min
+        (0..max_doc)
+            .filter(|doc_id| !delete_bitset.is_deleted(*doc_id))
+            .map(|doc_id| u32_reader.get(doc_id))
+            .minmax()
+            .into_option()
+    }
+}
+
+fn extract_fieldnorm_reader(segment_reader: &SegmentReader, field: Field) -> Result<U32FastFieldReader> {
+    Ok(segment_reader.get_fieldnorms_reader(field)?)
+}
+
+fn extract_fast_field_reader(segment_reader: &SegmentReader, field: Field) -> Result<U32FastFieldReader> {
+    segment_reader.get_fast_field_reader(field)
+}
+
 impl IndexMerger {
     pub fn open(schema: Schema, segments: &[Segment]) -> Result<IndexMerger> {
-        let mut readers = Vec::new();
+        let mut readers = vec!();
         let mut max_doc = 0;
         for segment in segments {
-            let reader = try!(SegmentReader::open(segment.clone()));
-            max_doc += reader.max_doc();
-            readers.push(reader);
+            if segment.meta().num_docs() > 0 {
+                let reader = SegmentReader::open(segment.clone())?;
+                max_doc += reader.num_docs();
+                readers.push(reader);
+            }
         }
         Ok(IndexMerger {
             schema: schema,
@@ -63,74 +96,101 @@ impl IndexMerger {
         })
     }
 
-
-    fn write_fieldnorms(&self, fast_field_serializer: &mut FastFieldSerializer) -> Result<()> {
-        // TODO make sure that works even if the field is never here.
-        for field in self.schema
+    fn write_fieldnorms(&self,
+        fast_field_serializer: &mut FastFieldSerializer) -> Result<()> {
+        let fieldnorm_fastfields: Vec<Field> = self.schema
                          .fields()
                          .iter()
                          .enumerate()
                          .filter(|&(_, field_entry)| field_entry.is_indexed())
-                         .map(|(field_id, _)| Field(field_id as u8)) {
-            let mut u32_readers = Vec::new();
-            let mut min_val = u32::min_value();
-            let mut max_val = 0;
-            for reader in &self.readers {
-                let u32_reader = try!(reader.get_fieldnorms_reader(field));
-                min_val = min(min_val, u32_reader.min_val());
-                max_val = max(max_val, u32_reader.max_val());
-                u32_readers.push((reader.max_doc(), u32_reader));
-            }
-            try!(fast_field_serializer.new_u32_fast_field(field, min_val, max_val));
-            for (max_doc, u32_reader) in u32_readers {
-                for doc_id in 0..max_doc {
-                    let val = u32_reader.get(doc_id);
-                    try!(fast_field_serializer.add_val(val));
-                }
-            }
-            try!(fast_field_serializer.close_field());
-        }
-        Ok(())
+                         .map(|(field_id, _)| Field(field_id as u8))
+                         .collect();
+        self.generic_write_fast_field(fieldnorm_fastfields, &extract_fieldnorm_reader, fast_field_serializer)
     }
 
     fn write_fast_fields(&self, fast_field_serializer: &mut FastFieldSerializer) -> Result<()> {
-        for field in self.schema
+        let fast_fields: Vec<Field> = self.schema
                          .fields()
                          .iter()
                          .enumerate()
                          .filter(|&(_, field_entry)| field_entry.is_u32_fast())
-                         .map(|(field_id, _)| Field(field_id as u8)) {
-            let mut u32_readers = Vec::new();
-            let mut min_val = u32::min_value();
-            let mut max_val = 0;
+                         .map(|(field_id, _)| Field(field_id as u8))
+                         .collect();
+        self.generic_write_fast_field(fast_fields, &extract_fast_field_reader, fast_field_serializer)
+    }
+
+
+    // used both to merge field norms and regular u32 fast fields.
+    fn generic_write_fast_field(&self,
+        fields: Vec<Field>,
+        field_reader_extractor: &Fn(&SegmentReader, Field) -> Result<U32FastFieldReader>,
+        fast_field_serializer: &mut FastFieldSerializer) -> Result<()> {
+        
+        for field in fields {
+            
+            let mut u32_readers = vec!();
+            let mut min_val = u32::max_value();
+            let mut max_val = u32::min_value();
+            
             for reader in &self.readers {
-                let u32_reader = try!(reader.get_fast_field_reader(field));
-                min_val = min(min_val, u32_reader.min_val());
-                max_val = max(max_val, u32_reader.max_val());
-                u32_readers.push((reader.max_doc(), u32_reader));
-            }
-            try!(fast_field_serializer.new_u32_fast_field(field, min_val, max_val));
-            for (max_doc, u32_reader) in u32_readers {
-                for doc_id in 0..max_doc {
-                    let val = u32_reader.get(doc_id);
-                    try!(fast_field_serializer.add_val(val));
+                let u32_reader = field_reader_extractor(reader, field)?;
+                if let Some((seg_min_val, seg_max_val)) = compute_min_max_val(&u32_reader, reader.max_doc(), reader.delete_bitset()) {
+                    // the segment has some non-deleted documents
+                    min_val = min(min_val, seg_min_val);
+                    max_val = max(max_val, seg_max_val);
+                    u32_readers.push((reader.max_doc(), u32_reader, reader.delete_bitset()));
                 }
             }
+
+            if u32_readers.is_empty() {
+                // we have actually zero documents.
+                min_val = 0;
+                max_val = 0;
+            }
+
+            assert!(min_val <= max_val);
+            
+            try!(fast_field_serializer.new_u32_fast_field(field, min_val, max_val));
+            for (max_doc, u32_reader, delete_bitset) in u32_readers {
+                for doc_id in 0..max_doc {
+                    if !delete_bitset.is_deleted(doc_id) {
+                        let val = u32_reader.get(doc_id);
+                        try!(fast_field_serializer.add_val(val));
+                    }
+                }
+            }
+
             try!(fast_field_serializer.close_field());
         }
         Ok(())
     }
 
-    fn write_postings(&self, postings_serializer: &mut PostingsSerializer) -> Result<()> {
+    fn write_postings(&self,
+
+        postings_serializer: &mut PostingsSerializer) -> Result<()> {
+        
         let mut merged_terms = TermIterator::from(&self.readers[..]);
         let mut delta_position_computer = DeltaPositionComputer::new();
-        let mut offsets: Vec<DocId> = Vec::new();
+        
         let mut max_doc = 0;
-        for reader in &self.readers {
-            offsets.push(max_doc);
-            max_doc += reader.max_doc();
-        }
 
+        // map from segment doc ids to the resulting merged segment doc id.
+        let mut merged_doc_id_map: Vec<Vec<Option<DocId>>> = Vec::with_capacity(self.readers.len());
+        
+        for reader in &self.readers {
+            let mut segment_local_map = Vec::with_capacity(reader.max_doc() as usize);
+            for doc_id in 0..reader.max_doc() {
+                if reader.is_deleted(doc_id) {
+                    segment_local_map.push(None);
+                }
+                else {
+                    segment_local_map.push(Some(max_doc));
+                    max_doc += 1u32;
+                }
+            }
+            merged_doc_id_map.push(segment_local_map);
+        }
+        
         while merged_terms.advance() {
             // Create the total list of doc ids
             // by stacking the doc ids from the different segment.
@@ -142,41 +202,66 @@ impl IndexMerger {
             // - Segment 2's doc ids become  [seg0.max_doc + seg1.max_doc, seg0.max_doc + seg1.max_doc + seg2.max_doc]
             // ...
             let term = merged_terms.term();
-            let mut merged_postings =
-                ChainedPostings::from(
-                    merged_terms
-                        .segment_ords()
-                        .iter()
-                        .cloned()
-                        .flat_map(|segment_ord| {
-                            let offset = offsets[segment_ord];
-                            self.readers[segment_ord]
-                                .read_postings_all_info(&term)
-                                .map(|segment_postings| OffsetPostings::new(segment_postings, offset))
-                        })
-                        .collect::<Vec<_>>()
-            );
+            let mut term_written = false;
+            let segment_postings = merged_terms
+                    .segment_ords()
+                    .iter()
+                    .cloned()
+                    .flat_map(|segment_ord| {
+                        self.readers[segment_ord]
+                            .read_postings_all_info(&term)
+                            .map(|segment_postings| (segment_ord, segment_postings))
+                    })
+                    .collect::<Vec<_>>();
 
-            // We can now serialize this postings, by pushing each document to the
-            // postings serializer.
-            try!(postings_serializer.new_term(&term, merged_postings.len() as DocId));
-            while merged_postings.advance() {
-                let delta_positions: &[u32] =
-                    delta_position_computer.compute_delta_positions(merged_postings.positions());
-                try!(postings_serializer.write_doc(merged_postings.doc(),
-                                                   merged_postings.term_freq(),
-                                                   delta_positions));
+            // We can remove the term if all documents which
+            // contained it have been deleted.
+            if segment_postings.len() > 0 {
+
+                // We can now serialize this postings, by pushing each document to the
+                // postings serializer.                
+                
+                for (segment_ord, mut segment_postings) in segment_postings {
+                    let old_to_new_doc_id = &merged_doc_id_map[segment_ord];
+                    while segment_postings.advance() {
+                        if let Some(remapped_doc_id) = old_to_new_doc_id[segment_postings.doc() as usize] {
+                            if !term_written {
+                                // we make sure to only write the term iff
+                                // there is at least one document.
+                                postings_serializer.new_term(&term)?;
+                                term_written = true;
+                            }
+                            let delta_positions: &[u32] =
+                                delta_position_computer.compute_delta_positions(segment_postings.positions());
+                            try!(postings_serializer.write_doc(
+                                    remapped_doc_id,
+                                    segment_postings.term_freq(),
+                                    delta_positions));
+                        }
+                    }
+                }
+
+                if term_written {
+                    try!(postings_serializer.close_term());
+                }
             }
-            try!(postings_serializer.close_term());
+            
         }
-
         Ok(())
     }
 
     fn write_storable_fields(&self, store_writer: &mut StoreWriter) -> Result<()> {
         for reader in &self.readers {
             let store_reader = reader.get_store_reader();
-            try!(store_writer.stack_reader(store_reader));
+            for doc_id in 0..reader.max_doc() {
+                if !reader.is_deleted(doc_id) {
+                    let doc = try!(store_reader.get(doc_id));
+                    let field_values: Vec<&FieldValue> = doc.field_values()
+                        .iter()
+                        .collect();
+                    try!(store_writer.store(&field_values));
+                }
+            }   
         }
         Ok(())
     }
@@ -199,12 +284,17 @@ mod tests {
     use schema;
     use schema::Document;
     use schema::Term;
+    use query::TermQuery;
+    use schema::{Field, FieldValue};
     use core::Index;
+    use Searcher;
     use DocAddress;
     use collector::tests::FastFieldTestCollector;
     use collector::tests::TestCollector;
     use query::BooleanQuery;
+    use postings::SegmentPostingsOption;
     use schema::TextIndexingOptions;
+    use futures::Future;
 
     #[test]
     fn test_index_merger() {
@@ -239,7 +329,7 @@ mod tests {
                     doc.add_u32(score_field, 7);
                     index_writer.add_document(doc).unwrap();
                 }
-                index_writer.commit().unwrap();
+                index_writer.commit().expect("committed");
             }
 
             {
@@ -256,15 +346,19 @@ mod tests {
                     doc.add_u32(score_field, 13);
                     index_writer.add_document(doc).unwrap();
                 }
-                index_writer.commit().unwrap();
+                index_writer.commit().expect("Commit failed");
             }
         }
         {
-            let segments = index.searchable_segments();
+            let segment_ids = index.searchable_segment_ids().expect("Searchable segments failed.");
             let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
-            index_writer.merge(&segments).unwrap();
+            index_writer.merge(&segment_ids)
+                        .wait()
+                        .expect("Merging failed");
+            index_writer.wait_merging_threads().unwrap();
         }
         {
+            index.load_searchers().unwrap();
             let searcher = index.searcher();
             let get_doc_ids = |terms: Vec<Term>| {
                 let mut collector = TestCollector::default();
@@ -307,11 +401,190 @@ mod tests {
                     let query = BooleanQuery::new_multiterms_query(terms);
                     let mut collector = FastFieldTestCollector::for_field(score_field);
                     assert!(searcher.search(&query, &mut collector).is_ok());
-                    collector.vals().clone()
+                    collector.vals()
                 };
                 assert_eq!(get_fast_vals(vec![Term::from_field_text(text_field, "a")]),
                            vec!(5, 7, 13,));
             }
         }
     }
+
+    fn search_term(searcher: &Searcher, term: Term) ->  Vec<u32> {
+        let mut collector = FastFieldTestCollector::for_field(Field(1));
+        let term_query = TermQuery::new(term, SegmentPostingsOption::NoFreq);
+        searcher.search(&term_query, &mut collector).unwrap();
+        collector.vals()
+    }
+
+    #[test]
+    fn test_index_merger_with_deletes() {
+        let mut schema_builder = schema::SchemaBuilder::default();
+        let text_fieldtype = schema::TextOptions::default()
+                                 .set_indexing_options(TextIndexingOptions::TokenizedWithFreq)
+                                 .set_stored();
+        let text_field = schema_builder.add_text_field("text", text_fieldtype);
+        let score_fieldtype = schema::U32Options::default().set_fast();
+        let score_field = schema_builder.add_u32_field("score", score_fieldtype);
+        let index = Index::create_in_ram(schema_builder.build());
+        let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            
+        {   // a first commit
+            index_writer.add_document(
+                doc!(
+                    text_field => "a b d",
+                    score_field => 1
+                )).unwrap();
+            index_writer.add_document(
+                doc!(
+                    text_field => "b c",
+                    score_field => 2
+                )).unwrap();
+            index_writer.delete_term(Term::from_field_text(text_field, "c"));
+            index_writer.add_document(
+                doc!(
+                    text_field => "c d",
+                    score_field => 3
+                )).unwrap();
+            index_writer.commit().expect("committed");
+            index.load_searchers().unwrap();
+            let ref searcher = *index.searcher();
+            assert_eq!(searcher.num_docs(), 2);
+            assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
+            assert_eq!(searcher.segment_readers()[0].max_doc(), 3);
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "a")), vec!(1));
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "b")), vec!(1));
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "c")), vec!(3));
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "d")), vec!(1, 3));
+        }
+        {   // a second commit
+            index_writer.add_document(
+                doc!(
+                    text_field => "a d e",
+                    score_field => 4_000
+                )).unwrap();
+            index_writer.add_document(
+                doc!(
+                    text_field => "e f",
+                    score_field => 5_000
+                )).unwrap();
+            index_writer.delete_term(Term::from_field_text(text_field, "a"));
+            index_writer.delete_term(Term::from_field_text(text_field, "f"));
+            index_writer.add_document(
+                doc!(
+                    text_field => "f g",
+                    score_field => 6_000
+                )).unwrap();
+            index_writer.add_document(
+                doc!(
+                    text_field => "g h",
+                    score_field => 7_000
+                )).unwrap();
+            index_writer.commit().expect("committed");
+            index.load_searchers().unwrap();
+            let searcher = index.searcher();
+            assert_eq!(searcher.segment_readers().len(), 2);
+            assert_eq!(searcher.num_docs(), 3);
+            assert_eq!(searcher.segment_readers()[0].num_docs(), 1);
+            assert_eq!(searcher.segment_readers()[0].max_doc(), 3);
+            assert_eq!(searcher.segment_readers()[1].num_docs(), 2);
+            assert_eq!(searcher.segment_readers()[1].max_doc(), 4);
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "a")), vec!());
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "b")), vec!());
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "c")), vec!(3));
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "d")), vec!(3));
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "e")), vec!());
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "f")), vec!(6_000));
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "g")), vec!(6_000, 7_000));
+            
+            let score_field_reader = searcher.segment_reader(0).get_fast_field_reader(score_field).unwrap();
+            assert_eq!(score_field_reader.min_val(), 1);
+            assert_eq!(score_field_reader.max_val(), 3);
+
+            let score_field_reader = searcher.segment_reader(1).get_fast_field_reader(score_field).unwrap();
+            assert_eq!(score_field_reader.min_val(), 4000);
+            assert_eq!(score_field_reader.max_val(), 7000);
+        }
+        {   // merging the segments
+            let segment_ids = index.searchable_segment_ids().expect("Searchable segments failed.");
+            index_writer.merge(&segment_ids)
+                        .wait()
+                        .expect("Merging failed");
+            index.load_searchers().unwrap();
+            let ref searcher = *index.searcher();
+            assert_eq!(searcher.segment_readers().len(), 1);
+            assert_eq!(searcher.num_docs(), 3);
+            assert_eq!(searcher.segment_readers()[0].num_docs(), 3);
+            assert_eq!(searcher.segment_readers()[0].max_doc(), 3);
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "a")), vec!());
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "b")), vec!());
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "c")), vec!(3));
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "d")), vec!(3));
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "e")), vec!());
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "f")), vec!(6_000));
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "g")), vec!(6_000, 7_000));
+            let score_field_reader = searcher.segment_reader(0).get_fast_field_reader(score_field).unwrap();
+            assert_eq!(score_field_reader.min_val(), 3);
+            assert_eq!(score_field_reader.max_val(), 7000);
+        }
+        {   
+            // test a commit with only deletes
+            index_writer.delete_term(Term::from_field_text(text_field, "c"));
+            index_writer.commit().unwrap();
+        
+            index.load_searchers().unwrap();
+            let ref searcher = *index.searcher();
+            assert_eq!(searcher.segment_readers().len(), 1);
+            assert_eq!(searcher.num_docs(), 2);
+            assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
+            assert_eq!(searcher.segment_readers()[0].max_doc(), 3);
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "a")), vec!());
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "b")), vec!());
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "c")), vec!());
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "d")), vec!());
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "e")), vec!());
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "f")), vec!(6_000));
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "g")), vec!(6_000, 7_000));
+            let score_field_reader = searcher.segment_reader(0).get_fast_field_reader(score_field).unwrap();
+            assert_eq!(score_field_reader.min_val(), 3);
+            assert_eq!(score_field_reader.max_val(), 7000);
+        }
+        {   // Test merging a single segment in order to remove deletes.
+            let segment_ids = index.searchable_segment_ids().expect("Searchable segments failed.");
+            index_writer.merge(&segment_ids)
+                        .wait()
+                        .expect("Merging failed");
+            index.load_searchers().unwrap();
+
+            let ref searcher = *index.searcher();
+            assert_eq!(searcher.segment_readers().len(), 1);
+            assert_eq!(searcher.num_docs(), 2);
+            assert_eq!(searcher.segment_readers()[0].num_docs(), 2);
+            assert_eq!(searcher.segment_readers()[0].max_doc(), 2);
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "a")), vec!());
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "b")), vec!());
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "c")), vec!());
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "d")), vec!());
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "e")), vec!());
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "f")), vec!(6_000));
+            assert_eq!(search_term(&searcher, Term::from_field_text(text_field, "g")), vec!(6_000, 7_000));
+            let score_field_reader = searcher.segment_reader(0).get_fast_field_reader(score_field).unwrap();
+            assert_eq!(score_field_reader.min_val(), 6000);
+            assert_eq!(score_field_reader.max_val(), 7000);
+        }
+
+        {   // Test removing all docs
+            index_writer.delete_term(Term::from_field_text(text_field, "g"));
+            let segment_ids = index.searchable_segment_ids().expect("Searchable segments failed.");
+            index_writer.merge(&segment_ids)
+                        .wait()
+                        .expect("Merging failed");
+            index.load_searchers().unwrap();
+
+            let ref searcher = *index.searcher();
+            assert_eq!(searcher.segment_readers().len(), 1);
+            assert_eq!(searcher.num_docs(), 0);
+        }
+        
+        
+    }
 }

src/indexer/mod.rs

@@ -7,9 +7,17 @@ mod log_merge_policy;
 mod segment_register;
 mod segment_writer;
 mod segment_manager;
+pub mod delete_queue;
 pub mod segment_updater;
 mod directory_lock;
+mod segment_entry;
+mod doc_opstamp_mapping;
+pub mod operation;
 
+
+// TODO avoid exposing SegmentState / SegmentEntry if it does not have to be public API
+
+pub use self::segment_entry::{SegmentEntry, SegmentState};
 pub use self::segment_serializer::SegmentSerializer;
 pub use self::segment_writer::SegmentWriter;
 pub use self::index_writer::IndexWriter;
@@ -17,6 +25,5 @@ pub use self::log_merge_policy::LogMergePolicy;
 pub use self::merge_policy::{NoMergePolicy, MergeCandidate, MergePolicy};
 pub use self::segment_manager::SegmentManager;
 
-
 /// Alias for the default merge policy, which is the LogMergePolicy.
 pub type DefaultMergePolicy = LogMergePolicy;

src/indexer/operation.rs

@@ -0,0 +1,17 @@
+use schema::Document;
+use schema::Term;
+
+
+/// Timestamped Delete operation.
+#[derive(Clone, Eq, PartialEq, Debug)]
+pub struct DeleteOperation {
+    pub opstamp: u64,
+    pub term: Term,
+}
+
+/// Timestamped Add operation.
+#[derive(Eq, PartialEq, Debug)]
+pub struct AddOperation {
+    pub opstamp: u64,
+    pub document: Document,
+}

src/indexer/segment_entry.rs

@@ -0,0 +1,71 @@
+use indexer::doc_opstamp_mapping::DocToOpstampMapping;
+use core::SegmentMeta;
+use core::SegmentId;
+use std::fmt;
+
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+pub enum SegmentState {
+    Ready,
+    InMerge,    
+}
+
+impl SegmentState {
+    pub fn letter_code(&self,) -> char {
+        match *self {
+            SegmentState::InMerge => 'M',
+            SegmentState::Ready => 'R',
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct SegmentEntry {
+    meta: SegmentMeta,
+    state: SegmentState,
+    doc_to_opstamp: DocToOpstampMapping,
+}
+
+impl SegmentEntry {
+
+    pub fn new(segment_meta: SegmentMeta) -> SegmentEntry {
+        SegmentEntry {
+            meta: segment_meta,
+            state: SegmentState::Ready,
+            doc_to_opstamp: DocToOpstampMapping::None,
+        }
+    }
+
+    pub fn doc_to_opstamp(&self) -> &DocToOpstampMapping {
+        &self.doc_to_opstamp
+    }
+
+    pub fn state(&self) -> SegmentState {
+        self.state
+    }
+
+    pub fn set_doc_to_opstamp(&mut self, doc_to_opstamp: DocToOpstampMapping) {
+        self.doc_to_opstamp = doc_to_opstamp;
+    }
+
+    pub fn segment_id(&self) -> SegmentId {
+        self.meta.id()
+    }
+    
+    pub fn meta(&self) -> &SegmentMeta {
+        &self.meta
+    }
+
+    pub fn start_merge(&mut self,) {
+        self.state = SegmentState::InMerge;
+    }
+
+    pub fn is_ready(&self,) -> bool {
+        self.state == SegmentState::Ready
+    }
+}
+
+impl fmt::Debug for SegmentEntry {
+    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        write!(formatter, "SegmentEntry({:?}, {:?})", self.meta, self.state)
+    }
+}

src/indexer/segment_manager.rs

@@ -2,27 +2,18 @@ use super::segment_register::SegmentRegister;
 use std::sync::RwLock;
 use core::SegmentMeta;
 use core::SegmentId;
+use indexer::SegmentEntry;
 use std::sync::{RwLockReadGuard, RwLockWriteGuard};
 use std::fmt::{self, Debug, Formatter};
-use std::sync::atomic::{AtomicUsize, Ordering};
 
 struct SegmentRegisters {
-    docstamp: u64,
     uncommitted: SegmentRegister,
     committed: SegmentRegister,
 }
 
-#[derive(Eq, PartialEq)]
-pub enum CommitState {
-    Committed,
-    Uncommitted,
-    Missing,
-}
-
 impl Default for SegmentRegisters {
     fn default() -> SegmentRegisters {
         SegmentRegisters {
-            docstamp: 0u64,
             uncommitted: SegmentRegister::default(),
             committed: SegmentRegister::default()
         }
@@ -37,12 +28,6 @@ impl Default for SegmentRegisters {
 /// changes (merges especially)
 pub struct SegmentManager {
     registers: RwLock<SegmentRegisters>,
-    // generation  is an ever increasing counter that 
-    // is incremented whenever we modify 
-    // the segment manager. It can be useful for debugging
-    // purposes, and it also acts as a "dirty" marker,
-    // to detect when the `meta.json` should be written.
-    generation: AtomicUsize, 
 }
 
 impl Debug for SegmentManager {
@@ -58,43 +43,43 @@ impl Debug for SegmentManager {
 ///
 /// For instance, a segment will not appear in both committed and uncommitted 
 /// segments
-pub fn get_segment_ready_for_commit(segment_manager: &SegmentManager,) -> (Vec<SegmentMeta>, Vec<SegmentMeta>) {
+pub fn get_segments(segment_manager: &SegmentManager,) -> (Vec<SegmentMeta>, Vec<SegmentMeta>) {
     let registers_lock = segment_manager.read();
-    (registers_lock.committed.get_segment_ready_for_commit(),
-     registers_lock.uncommitted.get_segment_ready_for_commit())
+    (registers_lock.committed.get_segments(),
+     registers_lock.uncommitted.get_segments())
 }
 
 impl SegmentManager {
     
-    /// Returns whether a segment is committed, uncommitted or missing.
-    pub fn is_committed(&self, segment_id: SegmentId) -> CommitState {
-        let lock = self.read();
-        if lock.uncommitted.contains(segment_id) {
-            CommitState::Uncommitted
-        }
-        else if lock.committed.contains(segment_id) {
-            CommitState::Committed
-        }
-        else {
-            CommitState::Missing
-        }
-    }
-    
-    pub fn docstamp(&self,) -> u64 {
-        self.read().docstamp
-    }
-
     pub fn from_segments(segment_metas: Vec<SegmentMeta>) -> SegmentManager {
         SegmentManager {
-            registers: RwLock::new( SegmentRegisters {
-                docstamp: 0u64, // TODO put the actual value
+            registers: RwLock::new(SegmentRegisters {
                 uncommitted: SegmentRegister::default(),
-                committed: SegmentRegister::from(segment_metas),
+                committed: SegmentRegister::new(segment_metas),
             }),
-            generation: AtomicUsize::default(),
         }
     }
 
+    pub fn segment_entries(&self,) -> Vec<SegmentEntry> {
+        let mut segment_entries = self.read()
+            .uncommitted
+            .segment_entries();
+        segment_entries.extend(
+            self.read()
+            .committed
+            .segment_entries()
+        );
+        segment_entries
+    }
+    
+    pub fn segment_entry(&self, segment_id: &SegmentId) -> Option<SegmentEntry> {
+        let registers = self.read();
+        registers
+            .committed
+            .segment_entry(segment_id)
+            .or_else(|| registers.uncommitted.segment_entry(segment_id))        
+    }
+
     // Lock poisoning should never happen :
     // The lock is acquired and released within this class,
     // and the operations cannot panic. 
@@ -103,14 +88,9 @@ impl SegmentManager {
     }
 
     fn write(&self,) -> RwLockWriteGuard<SegmentRegisters> {
-        self.generation.fetch_add(1, Ordering::Release);
         self.registers.write().expect("Failed to acquire write lock on SegmentManager.")
     }
 
-    pub fn generation(&self,) -> usize {
-        self.generation.load(Ordering::Acquire)
-    }
-
     /// Removes all of the uncommitted segments
     /// and returns them.
     pub fn rollback(&self,) -> Vec<SegmentId> {
@@ -120,19 +100,13 @@ impl SegmentManager {
         segment_ids
     }
 
-    pub fn commit(&self, docstamp: u64) {
+    pub fn commit(&self, segment_entries: Vec<SegmentEntry>) {
         let mut registers_lock = self.write();
-        let segment_entries = registers_lock.uncommitted.segment_entries();
+        registers_lock.committed.clear();
+        registers_lock.uncommitted.clear();
         for segment_entry in segment_entries {
             registers_lock.committed.add_segment_entry(segment_entry);
         }
-        registers_lock.docstamp = docstamp;
-        registers_lock.uncommitted.clear();    
-    }
-    
-    pub fn add_segment(&self, segment_meta: SegmentMeta) {
-        let mut registers_lock = self.write();
-        registers_lock.uncommitted.add_segment(segment_meta);
     }
     
     pub fn start_merge(&self, segment_ids: &[SegmentId]) {
@@ -148,33 +122,45 @@ impl SegmentManager {
             }
         }
     }
-    
-    pub fn end_merge(&self, merged_segment_ids: &[SegmentId], merged_segment_meta: &SegmentMeta) {
+
+    pub fn add_segment(&self, segment_entry: SegmentEntry) {
         let mut registers_lock = self.write();
-        if registers_lock.uncommitted.contains_all(merged_segment_ids) {
-            for segment_id in merged_segment_ids {
+        registers_lock.uncommitted.add_segment_entry(segment_entry);
+    }
+    
+    pub fn end_merge(&self, merged_segment_metas: &[SegmentMeta], merged_segment_entry: SegmentEntry) {
+        let mut registers_lock = self.write();
+        let merged_segment_ids: Vec<SegmentId> = merged_segment_metas.iter().map(|meta| meta.id()).collect();
+        if registers_lock.uncommitted.contains_all(&merged_segment_ids) {
+            for segment_id in &merged_segment_ids {
                 registers_lock.uncommitted.remove_segment(segment_id);
             }
-            registers_lock.uncommitted.add_segment(merged_segment_meta.clone());
+            registers_lock.uncommitted.add_segment_entry(merged_segment_entry);
         }
-        else if registers_lock.committed.contains_all(merged_segment_ids) {
-            for segment_id in merged_segment_ids {
+        else if registers_lock.committed.contains_all(&merged_segment_ids) {
+            for segment_id in &merged_segment_ids {
                 registers_lock.committed.remove_segment(segment_id);
             }
-            registers_lock.committed.add_segment(merged_segment_meta.clone());
+            registers_lock.committed.add_segment_entry(merged_segment_entry);
         } else {
             warn!("couldn't find segment in SegmentManager");
         }
     }
-    
-    pub fn committed_segments(&self,) -> Vec<SegmentId> {
+
+    pub fn committed_segment_metas(&self,) -> Vec<SegmentMeta> {
         let registers_lock = self.read();
-        registers_lock.committed.segment_ids()
-    }
-    
-    pub fn segment_metas(&self,) -> (Vec<SegmentMeta>, Vec<SegmentMeta>) {
-        let registers_lock = self.read();
-        (registers_lock.committed.segment_metas(), registers_lock.uncommitted.segment_metas())
+        registers_lock.committed.segment_metas()
     }
 }
 
+
+impl Default for SegmentManager {
+    fn default() -> SegmentManager {
+        SegmentManager {
+            registers: RwLock::new( SegmentRegisters {
+                uncommitted: SegmentRegister::default(),
+                committed: SegmentRegister::default(),
+            }),
+        }
+    }
+}

src/indexer/segment_register.rs

@@ -3,37 +3,7 @@ use std::collections::HashMap;
 use core::SegmentMeta;
 use std::fmt;
 use std::fmt::{Debug, Formatter};
-
-#[derive(Clone, PartialEq, Eq, Debug)]
-pub enum SegmentState {
-    Ready,
-    InMerge,    
-}
-
-impl SegmentState {
-    fn letter_code(&self,) -> char {
-        match *self {
-            SegmentState::InMerge => 'M',
-            SegmentState::Ready => 'R',
-        }
-    }
-}
-
-#[derive(Clone)]
-pub struct SegmentEntry {
-    meta: SegmentMeta,
-    state: SegmentState,
-}
-
-impl SegmentEntry {
-    fn start_merge(&mut self,) {
-        self.state = SegmentState::InMerge;
-    }
-
-    fn is_ready(&self,) -> bool {
-        self.state == SegmentState::Ready
-    }
-}
+use indexer::segment_entry::SegmentEntry;
 
 
 
@@ -49,11 +19,12 @@ pub struct SegmentRegister {
     segment_states: HashMap<SegmentId, SegmentEntry>, 
 }
 
+
 impl Debug for SegmentRegister {
     fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {
         try!(write!(f, "SegmentRegister("));
         for (k, v) in &self.segment_states {
-            try!(write!(f, "{}:{}, ", k.short_uuid_string(), v.state.letter_code()));
+            try!(write!(f, "{}:{}, ", k.short_uuid_string(), v.state().letter_code()));
         }
         try!(write!(f, ")"));
         Ok(())
@@ -66,15 +37,15 @@ impl SegmentRegister {
         self.segment_states.clear();
     }
     
-    pub fn get_segment_ready_for_commit(&self,) -> Vec<SegmentMeta> {
+    pub fn get_segments(&self,) -> Vec<SegmentMeta> {
         self.segment_states
             .values()
             .filter(|segment_entry| segment_entry.is_ready())
-            .map(|segment_entry| segment_entry.meta.clone())
+            .map(|segment_entry| segment_entry.meta().clone())
             .collect()
     }
     
-    pub fn segment_entries(&self,) -> Vec<SegmentEntry>{
+    pub fn segment_entries(&self,) -> Vec<SegmentEntry> {
         self.segment_states
             .values()
             .cloned()
@@ -84,31 +55,25 @@ impl SegmentRegister {
     pub fn segment_metas(&self,) -> Vec<SegmentMeta> {
         let mut segment_ids: Vec<SegmentMeta> = self.segment_states
             .values()
-            .map(|segment_entry| segment_entry.meta.clone())
+            .map(|segment_entry| segment_entry.meta().clone())
             .collect();
-        segment_ids.sort_by_key(|meta| meta.segment_id);
+        segment_ids.sort_by_key(|meta| meta.id());
         segment_ids
     }
     
     pub fn segment_ids(&self,) -> Vec<SegmentId> {
         self.segment_metas()
             .into_iter()
-            .map(|segment_meta| segment_meta.segment_id)
+            .map(|segment_meta| segment_meta.id())
             .collect()
     }
     
-    #[cfg(test)]
     pub fn segment_entry(&self, segment_id: &SegmentId) -> Option<SegmentEntry> {
         self.segment_states
             .get(&segment_id)
             .map(|segment_entry| segment_entry.clone())
     }
-
-    pub fn contains(&self, segment_id: SegmentId) -> bool {
-        self.segment_states.contains_key(&segment_id)
-    }
-
-
+    
     pub fn contains_all(&mut self, segment_ids: &[SegmentId]) -> bool {
         segment_ids
             .iter()
@@ -116,17 +81,10 @@ impl SegmentRegister {
     }
     
     pub fn add_segment_entry(&mut self, segment_entry: SegmentEntry) {
-        let segment_id = segment_entry.meta.segment_id;
+        let segment_id = segment_entry.segment_id();
         self.segment_states.insert(segment_id, segment_entry);
     }
     
-    pub fn add_segment(&mut self, segment_meta: SegmentMeta) {
-        self.add_segment_entry(SegmentEntry {
-            meta: segment_meta.clone(),
-            state: SegmentState::Ready,
-        });
-    }
-    
     pub fn remove_segment(&mut self, segment_id: &SegmentId) {
         self.segment_states.remove(segment_id);
     }   
@@ -138,24 +96,16 @@ impl SegmentRegister {
             .start_merge();
     } 
     
-    
-}
-
-
-impl From<Vec<SegmentMeta>> for SegmentRegister {
-    fn from(segment_metas: Vec<SegmentMeta>) -> SegmentRegister {
-        let mut segment_states = HashMap::new();
-        for segment_meta in segment_metas {
-            let segment_id = segment_meta.segment_id;
-            let segment_entry = SegmentEntry {
-                meta: segment_meta,
-                state: SegmentState::Ready,
-                
-            };
-            segment_states.insert(segment_id, segment_entry);
-        }
+    pub fn new(segment_metas: Vec<SegmentMeta>) -> SegmentRegister {
         SegmentRegister {
-            segment_states: segment_states,
+            segment_states: segment_metas
+                .into_iter()
+                .map(|segment_meta| {
+                    let segment_id = segment_meta.id();
+                    let segment_entry = SegmentEntry::new(segment_meta  );
+                    (segment_id, segment_entry)
+                })
+                .collect(),
         }
     }
 }
@@ -170,7 +120,7 @@ impl Default for SegmentRegister {
 
 #[cfg(test)]
 mod tests {
-    
+    use indexer::SegmentState;
     use core::SegmentId;
     use core::SegmentMeta;
     use super::*;
@@ -181,19 +131,31 @@ mod tests {
         let segment_id_a = SegmentId::generate_random();
         let segment_id_b = SegmentId::generate_random();
         let segment_id_merged = SegmentId::generate_random();
-        let segment_meta_merged = SegmentMeta::new(segment_id_merged, 10 + 20);
-        segment_register.add_segment(SegmentMeta::new(segment_id_a, 10));
-        assert_eq!(segment_register.segment_entry(&segment_id_a).unwrap().state, SegmentState::Ready);
+        
+        {
+            let segment_meta = SegmentMeta::new(segment_id_a);
+            let segment_entry = SegmentEntry::new(segment_meta);
+            segment_register.add_segment_entry(segment_entry);
+        }
+        assert_eq!(segment_register.segment_entry(&segment_id_a).unwrap().state(), SegmentState::Ready);
         assert_eq!(segment_register.segment_ids(), vec!(segment_id_a));
-        segment_register.add_segment(SegmentMeta::new(segment_id_b, 20));
-        assert_eq!(segment_register.segment_entry(&segment_id_b).unwrap().state, SegmentState::Ready);
+        {
+            let segment_meta = SegmentMeta::new(segment_id_b);
+            let segment_entry = SegmentEntry::new(segment_meta);
+            segment_register.add_segment_entry(segment_entry);
+        }
+        assert_eq!(segment_register.segment_entry(&segment_id_b).unwrap().state(), SegmentState::Ready);
         segment_register.start_merge(&segment_id_a);
         segment_register.start_merge(&segment_id_b);
-        assert_eq!(segment_register.segment_entry(&segment_id_a).unwrap().state, SegmentState::InMerge);
-        assert_eq!(segment_register.segment_entry(&segment_id_b).unwrap().state, SegmentState::InMerge);
+        assert_eq!(segment_register.segment_entry(&segment_id_a).unwrap().state(), SegmentState::InMerge);
+        assert_eq!(segment_register.segment_entry(&segment_id_b).unwrap().state(), SegmentState::InMerge);
         segment_register.remove_segment(&segment_id_a);
         segment_register.remove_segment(&segment_id_b);
-        segment_register.add_segment(segment_meta_merged);        
+        {
+            let segment_meta_merged = SegmentMeta::new(segment_id_merged);
+            let segment_entry = SegmentEntry::new(segment_meta_merged);
+            segment_register.add_segment_entry(segment_entry);        
+        }
         assert_eq!(segment_register.segment_ids(), vec!(segment_id_merged));        
     }
     

src/indexer/segment_updater.rs

@@ -1,44 +1,40 @@
 #![allow(for_kv_map)]
 
-use chan;
 use core::Index;
-use std::sync::Mutex;
+use core::IndexMeta;
+use core::META_FILEPATH;
 use core::Segment;
 use core::SegmentId;
 use core::SegmentMeta;
-use std::mem;
 use core::SerializableSegment;
-use indexer::MergePolicy;
+use directory::Directory;
+use Error;
+use futures_cpupool::CpuPool;
+use futures::{Future, future};
+use futures::Canceled;
+use futures::oneshot;
+use indexer::{MergePolicy, DefaultMergePolicy};
+use indexer::delete_queue::DeleteQueue;
+use indexer::index_writer::advance_deletes;
 use indexer::MergeCandidate;
 use indexer::merger::IndexMerger;
+use indexer::SegmentEntry;
 use indexer::SegmentSerializer;
-use std::thread;
-use schema::Schema;
-use directory::Directory;
-use std::thread::JoinHandle;
-use std::sync::Arc;
-use std::collections::HashMap;
-use rustc_serialize::json;
 use Result;
-use core::IndexMeta;
-use core::META_FILEPATH;
+use rustc_serialize::json;
+use schema::Schema;
+use std::borrow::BorrowMut;
+use std::collections::HashMap;
 use std::io::Write;
-use super::segment_manager::{SegmentManager, get_segment_ready_for_commit};
-use super::super::core::index::get_segment_manager;
-
-pub type SegmentUpdateSender = chan::Sender<SegmentUpdate>;
-pub type SegmentUpdateReceiver = chan::Receiver<SegmentUpdate>;
-
-
-fn create_metas(segment_manager: &SegmentManager, schema: Schema, docstamp: u64) -> IndexMeta {
-    let (committed_segments, uncommitted_segments) = segment_manager.segment_metas();
-    IndexMeta {
-        committed_segments: committed_segments,
-        uncommitted_segments: uncommitted_segments,
-        schema: schema,
-        docstamp: docstamp,
-    }
-}
+use std::mem;
+use std::ops::DerefMut;
+use std::sync::Arc;
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering;
+use std::sync::RwLock;
+use std::thread;
+use std::thread::JoinHandle;
+use super::segment_manager::{SegmentManager, get_segments};
 
 
 /// Save the index meta file.
@@ -51,11 +47,10 @@ fn create_metas(segment_manager: &SegmentManager, schema: Schema, docstamp: u64)
 ///
 /// This method is not part of tantivy's public API
 pub fn save_new_metas(schema: Schema,
-                  docstamp: u64,
+                  opstamp: u64,
                   directory: &mut Directory)
                   -> Result<()> {
-    let segment_manager = SegmentManager::from_segments(Vec::new());
-    save_metas(&segment_manager, schema, docstamp, directory)
+    save_metas(vec!(), schema, opstamp, directory)
 }
 
 
@@ -69,305 +64,255 @@ pub fn save_new_metas(schema: Schema,
 /// and flushed.
 ///
 /// This method is not part of tantivy's public API
-pub fn save_metas(segment_manager: &SegmentManager,
+pub fn save_metas(segment_metas: Vec<SegmentMeta>,
                   schema: Schema,
-                  docstamp: u64,
+                  opstamp: u64,
                   directory: &mut Directory)
                   -> Result<()> {
-    let metas = create_metas(segment_manager, schema, docstamp);
+    let metas = IndexMeta {
+        segments: segment_metas,
+        schema: schema,
+        opstamp: opstamp,
+    };
     let mut w = Vec::new();
     try!(write!(&mut w, "{}\n", json::as_pretty_json(&metas)));
-    directory.atomic_write(&META_FILEPATH, &w[..])
-        .map_err(From::from)
-}
-
-
-#[derive(Debug, Clone)]
-pub enum SegmentUpdate {
-    
-    /// New segment added.
-    /// Created by the indexing worker thread 
-    AddSegment(SegmentMeta),
-    
-    /// A merge is ended.
-    /// Remove the merged segment and record the new 
-    /// large merged segment.
-    EndMerge(usize, Vec<SegmentId>, SegmentMeta),
-    
-    /// Happens when rollback is called.
-    /// The current generation of segments is cancelled.
-    CancelGeneration,
-    
-    /// Starts a new generation... This 
-    /// happens at the end of Rollback.
-    NewGeneration,
-    
-    /// Just dropping the Segment updater object
-    /// is safe, but some merge might be happening in 
-    /// the background and the user may want to wait for these 
-    /// threads to terminate.
-    /// 
-    /// When receiving the Terminate signal, the segment updater stops
-    /// receiving segment updates and just waits for the merging threads
-    /// to terminate.
-	Terminate,
-    
-    /// Commit marks uncommmitted segments as committed.
-	Commit(u64),
+    Ok(directory
+        .atomic_write(&META_FILEPATH, &w[..])?)
+        
 }
 
 
 
+// The segment update runner is in charge of processing all
+//  of the `SegmentUpdate`s.
+//
+// All this processing happens on a single thread
+// consuming a common queue. 
+#[derive(Clone)]
+pub struct SegmentUpdater(Arc<InnerSegmentUpdater>);
 
 
-/// The segment updater is in charge of processing all of the 
-/// `SegmentUpdate`s.
-///
-/// All this processing happens on a single thread
-/// consuming a common queue. 
-/// 
-/// The segment updates producers are :
-/// - indexing threads are sending new segments 
-/// - merging threads are sending merge operations
-/// - the index writer sends "terminate"
-pub struct SegmentUpdater {
-	index: Index,
-	is_cancelled_generation: bool,
-	segment_update_receiver: SegmentUpdateReceiver,
-	segment_update_sender: SegmentUpdateSender,
-    segment_manager_arc: Arc<SegmentManager>,
-    merge_policy: Arc<Mutex<Box<MergePolicy>>>,
-    merging_thread_id: usize,
-    merging_threads: HashMap<usize, JoinHandle<(Vec<SegmentId>, SegmentMeta)> >, 
+struct InnerSegmentUpdater {
+    pool: CpuPool,
+    index: Index,
+    segment_manager: SegmentManager,
+    merge_policy: RwLock<Box<MergePolicy>>,
+    merging_thread_id: AtomicUsize,
+    merging_threads: RwLock<HashMap<usize, JoinHandle<Result<SegmentEntry>>>>,
+    generation: AtomicUsize,
+    delete_queue: DeleteQueue,
 }
 
-
 impl SegmentUpdater {
-	    
-    pub fn start_updater(index: Index, merge_policy: Arc<Mutex<Box<MergePolicy>>>) -> (SegmentUpdateSender, JoinHandle<()>) {
-        let segment_updater = SegmentUpdater::new(index, merge_policy);
-        (segment_updater.segment_update_sender.clone(), segment_updater.start())
+
+    pub fn new(index: Index, delete_queue: DeleteQueue) -> Result<SegmentUpdater>
+    {   
+        let segments = index.segments()?;
+        let segment_manager = SegmentManager::from_segments(segments);
+        Ok(
+            SegmentUpdater(Arc::new(InnerSegmentUpdater {
+                pool: CpuPool::new(1),
+                index: index,
+                segment_manager: segment_manager,
+                merge_policy: RwLock::new(box DefaultMergePolicy::default()),
+                merging_thread_id: AtomicUsize::default(),
+                merging_threads: RwLock::new(HashMap::new()),
+                generation: AtomicUsize::default(),
+                delete_queue: delete_queue,
+            }))
+        )
     }
-    
-    fn new(index: Index, merge_policy: Arc<Mutex<Box<MergePolicy>>>) -> SegmentUpdater {
-        let segment_manager_arc = get_segment_manager(&index);
-        let (segment_update_sender, segment_update_receiver): (SegmentUpdateSender, SegmentUpdateReceiver) = chan::async();
-		SegmentUpdater {
-			index: index,
-            is_cancelled_generation: false,
-            segment_update_sender: segment_update_sender,
-            segment_update_receiver: segment_update_receiver,
-            segment_manager_arc: segment_manager_arc,
-            merge_policy: merge_policy,
-            merging_thread_id: 0,
-            merging_threads: HashMap::new(), 
-		}
-	}
-    
-    fn new_merging_thread_id(&mut self,) -> usize {
-        self.merging_thread_id += 1;
-        self.merging_thread_id
+
+    pub fn get_merge_policy(&self) -> Box<MergePolicy> {
+        self.0.merge_policy.read().unwrap().box_clone()
     }
-    
-    
-    fn end_merge(
-        &mut self,
-        segment_ids: Vec<SegmentId>,
-        segment_meta: SegmentMeta) {
-        
-        let segment_manager = self.segment_manager_arc.clone();
-        segment_manager.end_merge(&segment_ids, &segment_meta);
-        save_metas(
-            &*segment_manager,
-            self.index.schema(),
-            self.index.docstamp(),
-            self.index.directory_mut()).expect("Could not save metas.");
-        for segment_id in segment_ids {
-            self.index.delete_segment(segment_id);
+
+    pub fn set_merge_policy(&self, merge_policy: Box<MergePolicy>) {
+        *self.0.merge_policy.write().unwrap()= merge_policy;
+    }
+
+    fn get_merging_thread_id(&self) -> usize {
+        self.0.merging_thread_id.fetch_add(1, Ordering::SeqCst)
+    }
+
+
+    fn run_async<T: 'static + Send, F: 'static + Send + FnOnce(SegmentUpdater) -> T>(&self, f: F) -> impl Future<Item=T, Error=Error> {
+        let me_clone = self.clone();
+        self.0.pool.spawn_fn(move || {
+            Ok(f(me_clone))
+        })
+    }
+
+    pub fn rollback(&mut self, generation: usize) -> impl Future<Item=(), Error=Error> {
+        self.0.generation.store(generation, Ordering::Release);
+        self.run_async(|segment_updater| {
+            segment_updater.0.segment_manager.rollback();
+        })
+    }
+
+    pub fn add_segment(&self, generation: usize, segment_entry: SegmentEntry) -> impl Future<Item=bool, Error=Error> {
+        if generation >= self.0.generation.load(Ordering::Acquire) {
+            future::Either::A(self.run_async(|segment_updater| {
+                segment_updater.0.segment_manager.add_segment(segment_entry);
+                segment_updater.consider_merge_options();
+                true
+            }))
+        }
+        else {
+            future::Either::B(future::ok(false))
         }
     }
 
-
-    fn start_merges(&mut self,) {
-        
-        let merge_candidates = self.consider_merge_options();
-        
-        for MergeCandidate(segment_ids) in merge_candidates {
-            
-            let merging_thread_id = self.new_merging_thread_id();
-            
-            self.segment_manager().start_merge(&segment_ids);
-
-            let index_clone = self.index.clone();
-            let segment_update_sender_clone = self.segment_update_sender.clone();
-            
-            let merge_thread_handle = thread::Builder::new()
-                .name(format!("merge_thread_{:?}", merging_thread_id))
-                .spawn(move || {
-                    info!("Start merge: {:?}", segment_ids);
-                    let schema = index_clone.schema();
-                    let segments: Vec<Segment> = segment_ids
-                        .iter()
-                        .map(|&segment_id| index_clone.segment(segment_id))
-                        .collect();
-                    // An IndexMerger is like a "view" of our merged segments. 
-                    // TODO unwrap
-                    let merger: IndexMerger = IndexMerger::open(schema, &segments[..]).expect("Creating index merger failed");
-                    let mut merged_segment = index_clone.new_segment();
-                    // ... we just serialize this index merger in our new segment
-                    // to merge the two segments.
-                    let segment_serializer = SegmentSerializer::for_segment(&mut merged_segment).expect("Creating index serializer failed");
-                    let num_docs = merger.write(segment_serializer).expect("Serializing merged index failed");
-                    let segment_meta = SegmentMeta {
-                        segment_id: merged_segment.id(),
-                        num_docs: num_docs,
-                    };
-                    let segment_update = SegmentUpdate::EndMerge(merging_thread_id, segment_ids.clone(), segment_meta.clone());
-                    segment_update_sender_clone.send(segment_update.clone());
-                    (segment_ids, segment_meta)
-                })
-                .expect("Failed to spawn merge thread");
-            
-            self.merging_threads.insert(merging_thread_id, merge_thread_handle);
-        }
+    fn purge_deletes(&self) -> Result<Vec<SegmentMeta>> {
+        self.0.segment_manager
+            .segment_entries()
+            .into_iter()
+            .map(|segment_entry| {
+                let mut segment = self.0.index.segment(segment_entry.meta().clone()); 
+                advance_deletes(&mut segment, &self.0.delete_queue.snapshot(), segment_entry.doc_to_opstamp())
+            })
+            .collect()
     }
-    
-    fn consider_merge_options(&self,) -> Vec<MergeCandidate> {
-        let segment_manager = self.segment_manager();
-        let (committed_segments, uncommitted_segments) = get_segment_ready_for_commit(segment_manager);
+
+    pub fn commit(&self, opstamp: u64) -> impl Future<Item=(), Error=Error> {
+        self.run_async(move |segment_updater| {
+            let segment_metas = segment_updater.purge_deletes().expect("Failed purge deletes");
+            let segment_entries = segment_metas
+                .into_iter()
+                .map(SegmentEntry::new)
+                .collect::<Vec<_>>();
+            segment_updater.0.segment_manager.commit(segment_entries);
+            let mut directory = segment_updater.0.index.directory().box_clone();
+            save_metas(
+                    segment_updater.0.segment_manager.committed_segment_metas(),
+                    segment_updater.0.index.schema(),
+                    opstamp,
+                    directory.borrow_mut()).expect("Could not save metas.");
+            segment_updater.consider_merge_options();
+        })
+    }
+
+
+    pub fn start_merge(&self, segment_ids: &[SegmentId]) -> impl Future<Item=SegmentEntry, Error=Canceled> {
+        
+        self.0.segment_manager.start_merge(segment_ids);
+        let segment_updater_clone = self.clone();
+        
+        let segment_ids_vec = segment_ids.to_vec(); 
+        
+        let merging_thread_id = self.get_merging_thread_id();
+        let (merging_future_send, merging_future_recv) = oneshot();
+        
+        let delete_operations = self.0.delete_queue.snapshot();
+
+        if segment_ids.is_empty() {
+            return merging_future_recv;
+        }
+        
+        let merging_join_handle = thread::spawn(move || {
+            
+            // first we need to apply deletes to our segment.
+            info!("Start merge: {:?}", segment_ids_vec);
+            
+            let ref index = segment_updater_clone.0.index;
+            let schema = index.schema();
+
+            let mut segment_metas = vec!();
+            for segment_id in &segment_ids_vec {
+                if let Some(segment_entry) = segment_updater_clone.0
+                    .segment_manager
+                    .segment_entry(segment_id) {
+                    let mut segment = index.segment(segment_entry.meta().clone());
+                    let segment_meta = advance_deletes(
+                         &mut segment,
+                         &delete_operations,
+                         segment_entry.doc_to_opstamp())?;
+                    segment_metas.push(segment_meta);
+                }
+                else {
+                    error!("Error, had to abort merge as some of the segment is not managed anymore.a");
+                    return Err(Error::InvalidArgument(format!("Segment {:?} requested for merge is not managed.", segment_id)));
+                }
+            }
+            
+            let segments: Vec<Segment> = segment_metas
+                .iter()
+                .cloned()
+                .map(|segment_meta| index.segment(segment_meta))
+                .collect();
+            
+            // An IndexMerger is like a "view" of our merged segments. 
+            let merger: IndexMerger = IndexMerger::open(schema, &segments[..])?;
+            let mut merged_segment = index.new_segment(); 
+            
+            // ... we just serialize this index merger in our new segment
+            // to merge the two segments.
+            let segment_serializer = SegmentSerializer::for_segment(&mut merged_segment).expect("Creating index serializer failed");
+            let num_docs = merger.write(segment_serializer).expect("Serializing merged index failed");
+            let mut segment_meta = SegmentMeta::new(merged_segment.id());
+            segment_meta.set_num_docs(num_docs);
+            
+            let segment_entry = SegmentEntry::new(segment_meta);
+            segment_updater_clone
+                .end_merge(segment_metas.clone(), segment_entry.clone())
+                .wait()
+                .unwrap();
+            
+            merging_future_send.complete(segment_entry.clone());
+            segment_updater_clone.0.merging_threads.write().unwrap().remove(&merging_thread_id);
+            Ok(segment_entry)
+        });
+        self.0.merging_threads.write().unwrap().insert(merging_thread_id, merging_join_handle);
+        merging_future_recv
+    }
+
+
+    fn consider_merge_options(&self) {
+        let (committed_segments, uncommitted_segments) = get_segments(&self.0.segment_manager);
         // Committed segments cannot be merged with uncommitted_segments.
-        // We therefore consider merges using these two sets of segments independantly.
-        let merge_policy_lock = self.merge_policy.lock().unwrap();
-        let mut merge_candidates = merge_policy_lock.compute_merge_candidates(&uncommitted_segments);
-        let committed_merge_candidates = merge_policy_lock.compute_merge_candidates(&committed_segments);
+        // We therefore consider merges using these two sets of segments independently.
+        let merge_policy = self.get_merge_policy();
+        let mut merge_candidates = merge_policy.compute_merge_candidates(&uncommitted_segments);
+        let committed_merge_candidates = merge_policy.compute_merge_candidates(&committed_segments);
         merge_candidates.extend_from_slice(&committed_merge_candidates[..]);
-        merge_candidates
-    }
-    
-    
-	fn segment_manager(&self,) -> &SegmentManager {
-		&*self.segment_manager_arc
-	}
-    
-    pub fn start(self,) -> JoinHandle<()> {
-		thread::Builder::new()
-            .name("segment_update".to_string())
-            .spawn(move || {
-                self.process();
-		    })
-            .expect("Failed to start segment updater thread.")
-    }
-    
-	fn process(mut self,) {
-        
-        let segment_manager = self.segment_manager_arc.clone();
-        
-        for segment_update in self.segment_update_receiver.clone() {
-            
-            if let SegmentUpdate::Terminate = segment_update {
-                break;
-            }
-                
-            // we check the generation number as if it was 
-            // dirty-bit. If the value is different 
-            // to our generation, then the segment_manager has
-            // been update updated.
-            let generation_before_update = segment_manager.generation();
-            
-            self.process_one(segment_update);
-            
-            if generation_before_update != segment_manager.generation() {
-                // The segment manager has changed, we need to   
-                // - save meta.json	
-                save_metas(
-                    &*segment_manager,
-                    self.index.schema(),
-                    self.index.docstamp(),
-                    self.index.directory_mut()).expect("Could not save metas.");
-
-
-                // - update the searchers
-                
-                // update the searchers so that they eventually will
-                // use the new segments.
-                // TODO eventually have this work through watching meta.json
-                // so that an external process stays up to date as well. 
-                match self.index.load_searchers() {
-                    Ok(()) => {
-                    }
-                    Err(e) => {
-                        error!("Failure while loading new searchers {:?}", e);
-                        panic!(format!("Failure while loading new searchers {:?}", e));
-                    }
-                }
-                
-                // - start merges if required
-                self.start_merges();
-            }
-        }
-        
-        let mut merging_threads = HashMap::new();
-        mem::swap(&mut merging_threads, &mut self.merging_threads);
-        for (_, merging_thread_handle) in merging_threads {
-            match merging_thread_handle.join() {
-                Ok((segment_ids, segment_meta)) => {
-                    self.end_merge(segment_ids, segment_meta);
-                }
-                Err(e) => {
-                    error!("Error in merging thread {:?}", e);
-                    break;
-                } 
-            }
+        for MergeCandidate(segment_metas) in merge_candidates {
+            self.start_merge(&segment_metas);
         }
     }
 
     
-
-    
-	// Process a single segment update.
-	pub fn process_one(
-		&mut self,
-		segment_update: SegmentUpdate) {
-		
-		info!("Segment update: {:?}", segment_update);
+    fn end_merge(&self, 
+        merged_segment_metas: Vec<SegmentMeta>,
+        resulting_segment_entry: SegmentEntry) -> impl Future<Item=(), Error=Error> {
         
-		match segment_update {
-			SegmentUpdate::AddSegment(segment_meta) => {
-				if !self.is_cancelled_generation {
-					self.segment_manager().add_segment(segment_meta);
-				}
-				else {
-					// rollback has been called and this
-					// segment actually belong to the 
-					// documents that have been dropped.
-					//
-					// Let's just remove its files.
-					self.index.delete_segment(segment_meta.segment_id);
-				}
-			}
-			SegmentUpdate::EndMerge(merging_thread_id, segment_ids, segment_meta) => {
-                self.end_merge(
-                    segment_ids,
-                    segment_meta);
-                self.merging_threads.remove(&merging_thread_id);
-			}
-			SegmentUpdate::CancelGeneration => {
-				// Called during rollback. The segment 
-				// that will arrive will be ignored
-				// until a NewGeneration is update arrives.
-				self.is_cancelled_generation = true;
-			}
-			SegmentUpdate::NewGeneration => {
-				// After rollback, we can resume
-				// indexing new documents.
-				self.is_cancelled_generation = false;
-			}
-			SegmentUpdate::Commit(docstamp) => {
-				self.segment_manager().commit(docstamp);
-			}
-			SegmentUpdate::Terminate => {
-                panic!("We should have left the loop before processing it.");
-			}
-		}
-	}
+        self.run_async(move |segment_updater| {
+            segment_updater.0.segment_manager.end_merge(&merged_segment_metas, resulting_segment_entry);
+            let mut directory = segment_updater.0.index.directory().box_clone();
+            let segment_metas = segment_updater.0.segment_manager.committed_segment_metas();
+            save_metas(
+                segment_metas,
+                segment_updater.0.index.schema(),
+                segment_updater.0.index.opstamp(),
+                directory.borrow_mut()).expect("Could not save metas.");
+            for segment_meta in merged_segment_metas {
+                segment_updater.0.index.delete_segment(segment_meta.id());
+            }
+        })
+        
+    }
+
+    pub fn wait_merging_thread(&self) -> thread::Result<()> {
+        let mut new_merging_threads = HashMap::new();
+        {
+            let mut merging_threads = self.0.merging_threads.write().unwrap();
+            mem::swap(&mut new_merging_threads, merging_threads.deref_mut());
+        }
+        for (_, merging_thread_handle) in new_merging_threads {
+            merging_thread_handle
+                .join()
+                .map(|_| ())?
+        }
+        Ok(())
+    }
+
 }

src/indexer/segment_writer.rs

@@ -1,8 +1,7 @@
 use Result;
 use DocId;
 use std::io;
-use schema::Schema;
-use schema::Document;
+use schema::Schema;	
 use schema::Term;
 use core::SegmentInfo;
 use core::Segment;
@@ -19,6 +18,8 @@ use postings::{NothingRecorder, TermFrequencyRecorder, TFAndPositionRecorder};
 use indexer::segment_serializer::SegmentSerializer;
 use datastruct::stacker::Heap;
 use indexer::index_writer::MARGIN_IN_BYTES;
+use super::operation::AddOperation;
+
 
 /// A `SegmentWriter` is in charge of creating segment index from a
 /// documents.
@@ -32,6 +33,7 @@ pub struct SegmentWriter<'a> {
 	segment_serializer: SegmentSerializer,
 	fast_field_writers: U32FastFieldsWriter,
 	fieldnorms_writer: U32FastFieldsWriter,
+	doc_opstamps: Vec<u64>,
 }
 
 
@@ -80,7 +82,9 @@ impl<'a> SegmentWriter<'a> {
 	/// the flushing behavior as a buffer limit
 	/// - segment: The segment being written  
 	/// - schema
-	pub fn for_segment(heap: &'a Heap, mut segment: Segment, schema: &Schema) -> Result<SegmentWriter<'a>> {
+	pub fn for_segment(heap: &'a Heap,
+					   mut segment: Segment,
+					   schema: &Schema) -> Result<SegmentWriter<'a>> {
 		let segment_serializer = try!(SegmentSerializer::for_segment(&mut segment));
 		let mut per_field_postings_writers: Vec<Box<PostingsWriter + 'a>> = Vec::new();
 		for field_entry in schema.fields() {
@@ -94,6 +98,7 @@ impl<'a> SegmentWriter<'a> {
 			fieldnorms_writer: create_fieldnorms_writer(schema),
 			segment_serializer: segment_serializer,
 			fast_field_writers: U32FastFieldsWriter::from_schema(schema),
+			doc_opstamps: Vec::with_capacity(1_000),
 		})
 	}
 	
@@ -101,18 +106,18 @@ impl<'a> SegmentWriter<'a> {
 	/// 
 	/// Finalize consumes the `SegmentWriter`, so that it cannot 
 	/// be used afterwards.
-	pub fn finalize(mut self,) -> Result<()> {
+	pub fn finalize(mut self) -> Result<Vec<u64>> {
 		let segment_info = self.segment_info();
 		for per_field_postings_writer in &mut self.per_field_postings_writers {
 			per_field_postings_writer.close(self.heap);
 		}
-		try!(write(&self.per_field_postings_writers,
+		write(&self.per_field_postings_writers,
 			  &self.fast_field_writers,
 			  &self.fieldnorms_writer,
 			  segment_info,
 			  self.segment_serializer,
-			  self.heap));
-		Ok(())
+			  self.heap)?;
+		Ok(self.doc_opstamps)
 	}
 	
 	/// Returns true iff the segment writer's buffer has reached capacity.
@@ -125,12 +130,33 @@ impl<'a> SegmentWriter<'a> {
 	pub fn is_buffer_full(&self,) -> bool {
 		self.heap.num_free_bytes() <= MARGIN_IN_BYTES
 	}
+
+	// pub fn compute_doc_mapping_after_delete(&self, mut delete_queue_cursor: DeleteQueueCursor) -> Vec<Option<DocId>> {
+	// 	let delete_docs = self.compute_delete_mask(&mut delete_queue_cursor);
+	// 	let max_doc: usize = self.max_doc as usize;
+	// 	let mut doc_autoinc = 0u32;
+	// 	(0..max_doc)
+	// 	.map(|doc| {
+	// 		if delete_docs.contains(doc) {
+	// 			None
+	// 		}
+	// 		else {
+	// 			let new_doc = doc_autoinc;
+	// 			doc_autoinc += 1;
+	// 			Some(new_doc)
+	// 		}
+	// 	})
+	// 	.collect::<Vec<_>>()
+	// }
+	
 	
 	/// Indexes a new document
 	///
 	/// As a user, you should rather use `IndexWriter`'s add_document.
-    pub fn add_document(&mut self, doc: &Document, schema: &Schema) -> io::Result<()> {
+    pub fn add_document(&mut self, add_operation: &AddOperation, schema: &Schema) -> io::Result<()> {
         let doc_id = self.max_doc;
+		let doc = &add_operation.document;
+		self.doc_opstamps.push(add_operation.opstamp);
         for (field, field_values) in doc.get_sorted_field_values() {
 			let field_posting_writer: &mut Box<PostingsWriter> = &mut self.per_field_postings_writers[field.0 as usize];
 			let field_options = schema.get_field_entry(field);
@@ -165,7 +191,7 @@ impl<'a> SegmentWriter<'a> {
 			}
 		}
 		self.fieldnorms_writer.fill_val_up_to(doc_id);
-		self.fast_field_writers.add_document(doc);
+		self.fast_field_writers.add_document(&doc);
 		let stored_fieldvalues: Vec<&FieldValue> = doc
 			.field_values()
 			.iter()
@@ -215,7 +241,7 @@ fn write<'a>(per_field_postings_writers: &[Box<PostingsWriter + 'a>],
 		 segment_info: SegmentInfo,
 	  	 mut serializer: SegmentSerializer,
 		 heap: &'a Heap,) -> Result<u32> {
-		for per_field_postings_writer in per_field_postings_writers.iter() {
+		for per_field_postings_writer in per_field_postings_writers {
 			try!(per_field_postings_writer.serialize(serializer.get_postings_serializer(), heap));
 		}
 		try!(fast_field_writers.serialize(serializer.get_fast_field_serializer()));

src/lib.rs

@@ -45,7 +45,9 @@ extern crate combine;
 extern crate itertools;
 extern crate chan;
 extern crate crossbeam;
-
+extern crate bit_set;
+extern crate futures;
+extern crate futures_cpupool;
 
 #[cfg(feature="simdcompression")]
 extern crate libc;
@@ -115,14 +117,16 @@ pub use postings::SegmentPostingsOption;
 
 pub use core::TermIterator;
 
-#[cfg(feature="simdcompression")]
-pub fn version() -> &'static str {
-    concat!(version!(), "-simd")  
-}
 
-#[cfg(not(feature="simdcompression"))]
+/// Expose the current version of tantivy, as well
+/// whether it was compiled with the simd compression.
 pub fn version() -> &'static str {
-    concat!(version!(), "-nosimd") 
+    if cfg!(feature="simdcompression") {
+        concat!(version!(), "-simd")  
+    }
+    else {
+        concat!(version!(), "-nosimd") 
+    }
 }
 
 /// Tantivy's makes it possible to personalize when 
@@ -239,6 +243,7 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
+            index.load_searchers().unwrap();
             let searcher = index.searcher();
             let term_a = Term::from_field_text(text_field, "a");
             assert_eq!(searcher.doc_freq(&term_a), 3);
@@ -274,7 +279,7 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
-            
+            index.load_searchers().unwrap();
             let searcher = index.searcher();
             let segment_reader: &SegmentReader = searcher.segment_reader(0);
             let fieldnorms_reader = segment_reader.get_fieldnorms_reader(text_field).unwrap();
@@ -284,6 +289,143 @@ mod tests {
         }
     }
 
+
+    #[test]
+    fn test_delete_postings() {
+        let mut schema_builder = SchemaBuilder::default();
+        let text_field = schema_builder.add_text_field("text", TEXT);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        {
+            // writing the segment
+            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            {   // 0
+                let doc = doc!(text_field=>"a b");
+                index_writer.add_document(doc).unwrap();
+            }
+            {   // 1
+                let doc = doc!(text_field=>" a c");
+                index_writer.add_document(doc).unwrap();
+            }
+            {   // 2
+                let doc = doc!(text_field=>" b c");
+                index_writer.add_document(doc).unwrap();
+            }
+            {   // 3
+                let doc = doc!(text_field=>" b d");
+                index_writer.add_document(doc).unwrap();
+            }
+            {
+                index_writer.delete_term(Term::from_field_text(text_field, "c"));
+            }
+            {
+                index_writer.delete_term(Term::from_field_text(text_field, "a"));
+            }
+            {   // 4
+                let doc = doc!(text_field=>" b c");
+                index_writer.add_document(doc).unwrap();
+            }
+            {   // 5
+                let doc = doc!(text_field=>" a");
+                index_writer.add_document(doc).unwrap();
+            }
+            index_writer.commit().unwrap();
+        }
+        {
+            index.load_searchers().unwrap();
+            let searcher = index.searcher();
+            let reader = searcher.segment_reader(0);
+            assert!(reader.read_postings_all_info(&Term::from_field_text(text_field, "abcd")).is_none());
+            {
+                let mut postings = reader.read_postings_all_info(&Term::from_field_text(text_field, "a")).unwrap();
+                assert!(postings.advance());
+                assert_eq!(postings.doc(), 5);
+                assert!(!postings.advance());
+            }
+            {
+                let mut postings = reader.read_postings_all_info(&Term::from_field_text(text_field, "b")).unwrap();
+                assert!(postings.advance());
+                assert_eq!(postings.doc(), 3);
+                assert!(postings.advance());
+                assert_eq!(postings.doc(), 4);
+                assert!(!postings.advance());
+            }
+        }
+        {
+            // writing the segment
+            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            {   // 0
+                let doc = doc!(text_field=>"a b");
+                index_writer.add_document(doc).unwrap();
+            }
+            {   // 1
+                index_writer.delete_term(Term::from_field_text(text_field, "c"));
+            }
+            index_writer.rollback().unwrap();
+        }
+        {
+            index.load_searchers().unwrap();
+            let searcher = index.searcher();
+            let reader = searcher.segment_reader(0);
+            assert!(reader.read_postings_all_info(&Term::from_field_text(text_field, "abcd")).is_none());
+            {
+                let mut postings = reader.read_postings_all_info(&Term::from_field_text(text_field, "a")).unwrap();
+                assert!(postings.advance());
+                assert_eq!(postings.doc(), 5);
+                assert!(!postings.advance());
+            }
+            {
+                let mut postings = reader.read_postings_all_info(&Term::from_field_text(text_field, "b")).unwrap();
+                assert!(postings.advance());
+                assert_eq!(postings.doc(), 3);
+                assert!(postings.advance());
+                assert_eq!(postings.doc(), 4);
+                assert!(!postings.advance());
+            }
+        }
+        {
+            // writing the segment
+            let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();
+            { 
+                let doc = doc!(text_field=>"a b");
+                index_writer.add_document(doc).unwrap();
+            }
+            {   
+                index_writer.delete_term(Term::from_field_text(text_field, "c"));
+            }
+            index_writer.rollback().unwrap();
+            {   
+                index_writer.delete_term(Term::from_field_text(text_field, "a"));
+            }
+            index_writer.commit().unwrap();
+        }
+        {
+            index.load_searchers().unwrap();
+            let searcher = index.searcher();
+            let reader = searcher.segment_reader(0);
+            assert!(reader.read_postings_all_info(&Term::from_field_text(text_field, "abcd")).is_none());
+            {
+                let mut postings = reader.read_postings_all_info(&Term::from_field_text(text_field, "a")).unwrap();
+                assert!(!postings.advance());
+            }
+            {
+                let mut postings = reader.read_postings_all_info(&Term::from_field_text(text_field, "b")).unwrap();
+                assert!(postings.advance());
+                assert_eq!(postings.doc(), 3);
+                assert!(postings.advance());
+                assert_eq!(postings.doc(), 4);
+                assert!(!postings.advance());
+            }
+            {
+                let mut postings = reader.read_postings_all_info(&Term::from_field_text(text_field, "c")).unwrap();
+                assert!(postings.advance());
+                assert_eq!(postings.doc(), 4);
+                assert!(!postings.advance());
+            }
+        }
+    }
+
+
     #[test]
     fn test_termfreq() {
         let mut schema_builder = SchemaBuilder::default();
@@ -300,6 +442,7 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
+            index.load_searchers().unwrap();
             let searcher = index.searcher();
             let reader = searcher.segment_reader(0);
             assert!(reader.read_postings_all_info(&Term::from_field_text(text_field, "abcd")).is_none());
@@ -336,6 +479,7 @@ mod tests {
             index_writer.commit().unwrap();
         }
         {
+            index.load_searchers().unwrap();
             let searcher = index.searcher();
             let get_doc_ids = |terms: Vec<Term>| {
                 let query = BooleanQuery::new_multiterms_query(terms);

src/postings/chained_postings.rs

@@ -1,71 +0,0 @@
-use DocId;
-use postings::Postings;
-use postings::OffsetPostings;
-use postings::DocSet;
-use postings::HasLen;
-
-/// Creates a posting object that chains two postings 
-/// together.
-///
-/// When iterating over the chained postings,
-/// it will consume all of the documents of the first postings, 
-/// and then iterate over the documents over the second postings.
-/// 
-/// The chained postings is used when merging segments. 
-pub struct ChainedPostings<'a> {
-    chained_postings: Vec<OffsetPostings<'a>>,
-    posting_id: usize,
-    len: usize,
-}
-
-impl<'a> From<Vec<OffsetPostings<'a>>> for ChainedPostings<'a> {
-    fn from(chained_postings: Vec<OffsetPostings<'a>>) -> ChainedPostings {
-        let len: usize = chained_postings
-            .iter()
-            .map(|segment_postings| segment_postings.len())
-            .sum();
-        ChainedPostings {
-            chained_postings: chained_postings,
-            posting_id: 0,
-            len: len,
-        }
-    }
-}
-
-impl<'a> DocSet for ChainedPostings<'a> {
-
-    fn advance(&mut self,) -> bool {
-        if self.posting_id == self.chained_postings.len() {
-            return false;
-        }
-        while !self.chained_postings[self.posting_id].advance() {
-            self.posting_id += 1;
-            if self.posting_id == self.chained_postings.len() {
-                return false;
-            }   
-        }
-        true
-    }
-
-    fn doc(&self,) -> DocId {
-        self.chained_postings[self.posting_id].doc()
-    }
-}
-
-impl<'a> HasLen for ChainedPostings<'a> {
-    fn len(&self,) -> usize {
-        self.len
-    }
-}
-
-impl<'a> Postings for ChainedPostings<'a> {
-    
-    fn term_freq(&self,) -> u32 {
-        self.chained_postings[self.posting_id].term_freq()
-    }
-    
-    fn positions(&self) -> &[u32] {
-        self.chained_postings[self.posting_id].positions()
-    } 
-    
-}

src/postings/docset.rs

@@ -67,6 +67,7 @@ pub trait DocSet {
     }
 }
 
+
 impl<TDocSet: DocSet + ?Sized> DocSet for Box<TDocSet> {
     fn advance(&mut self) -> bool {
         let unboxed: &mut TDocSet = self.borrow_mut();

src/postings/intersection.rs

@@ -2,8 +2,6 @@ use postings::DocSet;
 use postings::SkipResult;
 use DocId;
 
-// TODO Find a way to specialize `IntersectionDocSet`
-
 /// Creates a `DocSet` that iterator through the intersection of two `DocSet`s.
 pub struct IntersectionDocSet<TDocSet: DocSet> {
     docsets: Vec<TDocSet>,

src/postings/mod.rs

@@ -9,17 +9,14 @@ mod recorder;
 mod serializer;
 mod postings_writer;
 mod term_info;
-mod chained_postings;
 mod vec_postings;
 mod segment_postings;
 mod intersection;
-mod offset_postings;
 mod freq_handler;
 mod docset;
 mod segment_postings_option;
 
 pub use self::docset::{SkipResult, DocSet};
-pub use self::offset_postings::OffsetPostings;
 pub use self::recorder::{Recorder, NothingRecorder, TermFrequencyRecorder, TFAndPositionRecorder};
 pub use self::serializer::PostingsSerializer;
 pub use self::postings_writer::PostingsWriter;
@@ -29,11 +26,9 @@ pub use self::postings::Postings;
 
 #[cfg(test)]
 pub use self::vec_postings::VecPostings;
-pub use self::chained_postings::ChainedPostings;
 pub use self::segment_postings::SegmentPostings;
 pub use self::intersection::IntersectionDocSet;
 pub use self::freq_handler::FreqHandler;
-
 pub use self::segment_postings_option::SegmentPostingsOption;
 pub use common::HasLen;
 
@@ -51,6 +46,7 @@ mod tests {
     use query::TermQuery;
     use schema::Field;
     use test::Bencher;
+    use indexer::operation::AddOperation;
     use rand::{XorShiftRng, Rng, SeedableRng};
     
         
@@ -63,7 +59,7 @@ mod tests {
         let mut segment = index.new_segment();
         let mut posting_serializer = PostingsSerializer::open(&mut segment).unwrap();
         let term = Term::from_field_text(text_field, "abc");
-        posting_serializer.new_term(&term, 3).unwrap();
+        posting_serializer.new_term(&term).unwrap();
         for doc_id in 0u32..3u32 {
             let positions = vec!(1,2,3,2);
             posting_serializer.write_doc(doc_id, 2, &positions).unwrap();
@@ -88,19 +84,31 @@ mod tests {
                 let mut doc = Document::default();
                 doc.add_text(text_field, "a b a c a d a a.");
                 doc.add_text(text_field, "d d d d a"); // checking that position works if the field has two values.
-                segment_writer.add_document(&doc, &schema).unwrap();
+                let op = AddOperation {
+                    opstamp: 0u64,
+                    document: doc,  
+                };
+                segment_writer.add_document(&op, &schema).unwrap();
             }
             {
                 let mut doc = Document::default();
                 doc.add_text(text_field, "b a");
-                segment_writer.add_document(&doc, &schema).unwrap();
+                let op = AddOperation {
+                    opstamp: 1u64,
+                    document: doc,  
+                };
+                segment_writer.add_document(&op, &schema).unwrap();
             }
             for i in 2..1000 {
                 let mut doc = Document::default();
                 let mut text = iter::repeat("e ").take(i).collect::<String>();
                 text.push_str(" a");
                 doc.add_text(text_field, &text);
-                segment_writer.add_document(&doc, &schema).unwrap();
+                let op = AddOperation {
+                    opstamp: 2u64,
+                    document: doc,  
+                };
+                segment_writer.add_document(&op, &schema).unwrap();
             }
             segment_writer.finalize().unwrap();
         }
@@ -176,6 +184,7 @@ mod tests {
             }
             assert!(index_writer.commit().is_ok());
         }
+        index.load_searchers().unwrap();
         let term_query = TermQuery::new(Term::from_field_text(text_field, "a"), SegmentPostingsOption::NoFreq);
         let searcher = index.searcher();
         let mut term_weight = term_query.specialized_weight(&*searcher);
@@ -252,6 +261,7 @@ mod tests {
                 }
                 assert!(index_writer.commit().is_ok());
             }
+            index.load_searchers().unwrap();
             index
         };
     }
@@ -271,7 +281,6 @@ mod tests {
     fn bench_segment_intersection(b: &mut Bencher) {
         let searcher = INDEX.searcher();
         let segment_reader = searcher.segment_reader(0);
-        
         b.iter(|| {
             let segment_postings_a = segment_reader.read_postings(&*TERM_A, SegmentPostingsOption::NoFreq).unwrap();
             let segment_postings_b = segment_reader.read_postings(&*TERM_B, SegmentPostingsOption::NoFreq).unwrap();

src/postings/offset_postings.rs

@@ -1,59 +0,0 @@
-use postings::Postings;
-use postings::SegmentPostings;
-use postings::SkipResult;
-use postings::DocSet;
-use postings::HasLen;
-use DocId;
-
-/// Wraps a posting object and offset all of the doc id with a given offset.
-///
-/// Assuming the original posting list is `0, 5, 7, 8...`, and the offset is `3`
-/// the `OffsetPostings` becomes  `3, 8, 10, 11...`.
-pub struct OffsetPostings<'a> {
-    underlying: SegmentPostings<'a>,
-    offset: DocId,
-}
-
-impl<'a> OffsetPostings<'a> {
-    /// Constructor
-    pub fn new(underlying: SegmentPostings<'a>, offset: DocId) -> OffsetPostings {
-        OffsetPostings {
-            underlying: underlying,
-            offset: offset,
-        }
-    }
-}
-
-impl<'a> DocSet for OffsetPostings<'a> {
-    fn advance(&mut self) -> bool {
-        self.underlying.advance()
-    }
-
-    fn doc(&self) -> DocId {
-        self.underlying.doc() + self.offset
-    }
-
-    fn skip_next(&mut self, target: DocId) -> SkipResult {
-        if target >= self.offset {
-            SkipResult::OverStep
-        } else {
-            self.underlying.skip_next(target - self.offset)
-        }
-    }
-}
-
-impl<'a> HasLen for OffsetPostings<'a> {
-    fn len(&self) -> usize {
-        self.underlying.len()
-    }
-}
-
-impl<'a> Postings for OffsetPostings<'a> {
-    fn term_freq(&self) -> u32 {
-        self.underlying.term_freq()
-    }
-
-    fn positions(&self) -> &[u32] {
-        self.underlying.positions()
-    }
-}

src/postings/postings_writer.rs

@@ -22,7 +22,7 @@ pub trait PostingsWriter {
     /// * heap - heap used to store the postings informations as well as the terms
     /// in the hashmap.
     fn suscribe(&mut self, doc: DocId, pos: u32, term: &Term, heap: &Heap);
-
+    
     /// Serializes the postings on disk.
     /// The actual serialization format is handled by the `PostingsSerializer`.
     fn serialize(&self, serializer: &mut PostingsSerializer, heap: &Heap) -> io::Result<()>;
@@ -99,6 +99,7 @@ impl<'a, Rec: Recorder + 'static> PostingsWriter for SpecializedPostingsWriter<'
         }
     }
 
+
     #[inline]
     fn suscribe(&mut self, doc: DocId, position: u32, term: &Term, heap: &Heap) {
         let mut recorder = self.term_index.get_or_create(term);
@@ -119,9 +120,9 @@ impl<'a, Rec: Recorder + 'static> PostingsWriter for SpecializedPostingsWriter<'
         term_offsets.sort_by_key(|&(k, _v)| k);
         let mut term = Term::allocate(Field(0), 100);
         for (term_bytes, (addr, recorder)) in term_offsets {
-            // TODO remove copy
+            // sadly we are required to copy the data
             term.set_content(term_bytes);
-            try!(serializer.new_term(&term, recorder.doc_freq()));
+            try!(serializer.new_term(&term));
             try!(recorder.serialize(addr, serializer, heap));
             try!(serializer.close_term());
         }

src/postings/recorder.rs

@@ -26,8 +26,6 @@ pub trait Recorder: HeapAllocable {
     fn record_position(&mut self, position: u32, heap: &Heap);
     /// Close the document. It will help record the term frequency.
     fn close_doc(&mut self, heap: &Heap);
-    /// Returns the number of document that have been seen so far
-    fn doc_freq(&self) -> u32;
     /// Pushes the postings information to the serializer.
     fn serialize(&self,
                  self_addr: u32,
@@ -41,7 +39,6 @@ pub trait Recorder: HeapAllocable {
 pub struct NothingRecorder {
     stack: ExpUnrolledLinkedList,
     current_doc: DocId,
-    doc_freq: u32,
 }
 
 impl HeapAllocable for NothingRecorder {
@@ -49,7 +46,6 @@ impl HeapAllocable for NothingRecorder {
         NothingRecorder {
             stack: ExpUnrolledLinkedList::with_addr(addr),
             current_doc: u32::max_value(),
-            doc_freq: 0u32,
         }
     }
 }
@@ -62,17 +58,12 @@ impl Recorder for NothingRecorder {
     fn new_doc(&mut self, doc: DocId, heap: &Heap) {
         self.current_doc = doc;
         self.stack.push(doc, heap);
-        self.doc_freq += 1;
     }
 
     fn record_position(&mut self, _position: u32, _heap: &Heap) {}
 
     fn close_doc(&mut self, _heap: &Heap) {}
 
-    fn doc_freq(&self) -> u32 {
-        self.doc_freq
-    }
-
     fn serialize(&self,
                  self_addr: u32,
                  serializer: &mut PostingsSerializer,
@@ -91,7 +82,6 @@ pub struct TermFrequencyRecorder {
     stack: ExpUnrolledLinkedList,
     current_doc: DocId,
     current_tf: u32,
-    doc_freq: u32,
 }
 
 impl HeapAllocable for TermFrequencyRecorder {
@@ -100,7 +90,6 @@ impl HeapAllocable for TermFrequencyRecorder {
             stack: ExpUnrolledLinkedList::with_addr(addr),
             current_doc: u32::max_value(),
             current_tf: 0u32,
-            doc_freq: 0u32,
         }
     }
 }
@@ -111,7 +100,6 @@ impl Recorder for TermFrequencyRecorder {
     }
 
     fn new_doc(&mut self, doc: DocId, heap: &Heap) {
-        self.doc_freq += 1u32;
         self.current_doc = doc;
         self.stack.push(doc, heap);
     }
@@ -126,10 +114,6 @@ impl Recorder for TermFrequencyRecorder {
         self.current_tf = 0;
     }
 
-    fn doc_freq(&self) -> u32 {
-        self.doc_freq
-    }
-
     fn serialize(&self,
                  self_addr: u32,
                  serializer: &mut PostingsSerializer,
@@ -154,7 +138,6 @@ impl Recorder for TermFrequencyRecorder {
 pub struct TFAndPositionRecorder {
     stack: ExpUnrolledLinkedList,
     current_doc: DocId,
-    doc_freq: u32,
 }
 
 impl HeapAllocable for TFAndPositionRecorder {
@@ -162,7 +145,6 @@ impl HeapAllocable for TFAndPositionRecorder {
         TFAndPositionRecorder {
             stack: ExpUnrolledLinkedList::with_addr(addr),
             current_doc: u32::max_value(),
-            doc_freq: 0u32,
         }
     }
 }
@@ -173,7 +155,6 @@ impl Recorder for TFAndPositionRecorder {
     }
 
     fn new_doc(&mut self, doc: DocId, heap: &Heap) {
-        self.doc_freq += 1;
         self.current_doc = doc;
         self.stack.push(doc, heap);
     }
@@ -186,10 +167,6 @@ impl Recorder for TFAndPositionRecorder {
         self.stack.push(POSITION_END, heap);
     }
 
-    fn doc_freq(&self) -> u32 {
-        self.doc_freq
-    }
-
     fn serialize(&self,
                  self_addr: u32,
                  serializer: &mut PostingsSerializer,

src/postings/segment_postings.rs

@@ -2,6 +2,7 @@ use compression::{NUM_DOCS_PER_BLOCK, BlockDecoder, VIntDecoder};
 use DocId;
 use postings::{Postings, FreqHandler, DocSet, HasLen};
 use std::num::Wrapping;
+use fastfield::delete::DeleteBitSet;
 
 
 const EMPTY_DATA: [u8; 0] = [0u8; 0];
@@ -18,6 +19,7 @@ pub struct SegmentPostings<'a> {
     freq_handler: FreqHandler,
     remaining_data: &'a [u8],
     cur: Wrapping<usize>,
+    delete_bitset: DeleteBitSet,
 }
 
 impl<'a> SegmentPostings<'a> {
@@ -41,7 +43,10 @@ impl<'a> SegmentPostings<'a> {
     /// * `data` - data array. The complete data is not necessarily used.
     /// * `freq_handler` - the freq handler is in charge of decoding
     ///   frequencies and/or positions
-    pub fn from_data(len: u32, data: &'a [u8], freq_handler: FreqHandler) -> SegmentPostings<'a> {
+    pub fn from_data(len: u32,
+                     data: &'a [u8],
+                     delete_bitset: &'a DeleteBitSet,
+                     freq_handler: FreqHandler) -> SegmentPostings<'a> {
         SegmentPostings {
             len: len as usize,
             doc_offset: 0,
@@ -49,6 +54,7 @@ impl<'a> SegmentPostings<'a> {
             freq_handler: freq_handler,
             remaining_data: data,
             cur: Wrapping(usize::max_value()),
+            delete_bitset: delete_bitset.clone(),
         }
     }
 
@@ -60,6 +66,7 @@ impl<'a> SegmentPostings<'a> {
             block_decoder: BlockDecoder::new(),
             freq_handler: FreqHandler::new_without_freq(),
             remaining_data: &EMPTY_DATA,
+            delete_bitset: DeleteBitSet::empty(),
             cur: Wrapping(usize::max_value()),
         }
     }
@@ -77,14 +84,18 @@ impl<'a> DocSet for SegmentPostings<'a> {
     // next needs to be called a first time to point to the correct element.
     #[inline]
     fn advance(&mut self) -> bool {
-        self.cur += Wrapping(1);
-        if self.cur.0 >= self.len {
-            return false;
+        loop {
+            self.cur += Wrapping(1);
+            if self.cur.0 >= self.len {
+                return false;
+            }
+            if self.index_within_block() == 0 {
+                self.load_next_block();
+            }
+            if !self.delete_bitset.is_deleted(self.doc()) {
+                return true;
+            }
         }
-        if self.index_within_block() == 0 {
-            self.load_next_block();
-        }
-        true
     }
 
     #[inline]

src/postings/serializer.rs

@@ -49,7 +49,7 @@ use common::BinarySerializable;
 /// A description of the serialization format is
 /// [available here](https://fulmicoton.gitbooks.io/tantivy-doc/content/inverted-index.html).
 pub struct PostingsSerializer {
-    terms_fst_builder: FstMapBuilder<WritePtr, TermInfo>, /* TODO find an alternative to work around the "move" */
+    terms_fst_builder: FstMapBuilder<WritePtr, TermInfo>,
     postings_write: WritePtr,
     positions_write: WritePtr,
     written_bytes_postings: usize,
@@ -63,6 +63,7 @@ pub struct PostingsSerializer {
     schema: Schema,
     text_indexing_options: TextIndexingOptions,
     term_open: bool,
+    current_term_info: TermInfo,
 }
 
 impl PostingsSerializer {
@@ -88,6 +89,7 @@ impl PostingsSerializer {
             schema: schema,
             text_indexing_options: TextIndexingOptions::Unindexed,
             term_open: false,
+            current_term_info: TermInfo::default(),
         })
     }
 
@@ -121,7 +123,7 @@ impl PostingsSerializer {
     /// * term - the term. It needs to come after the previous term according
     ///   to the lexicographical order.
     /// * doc_freq - return the number of document containing the term.
-    pub fn new_term(&mut self, term: &Term, doc_freq: DocId) -> io::Result<()> {
+    pub fn new_term(&mut self, term: &Term) -> io::Result<()> {
         if self.term_open {
             panic!("Called new_term, while the previous term was not closed.");
         }
@@ -131,13 +133,12 @@ impl PostingsSerializer {
         self.last_doc_id_encoded = 0;
         self.term_freqs.clear();
         self.position_deltas.clear();
-        let term_info = TermInfo {
-            doc_freq: doc_freq,
+        self.current_term_info = TermInfo {
+            doc_freq: 0,
             postings_offset: self.written_bytes_postings as u32,
             positions_offset: self.written_bytes_positions as u32,
         };
-        self.terms_fst_builder
-            .insert(term.as_slice(), &term_info)
+        self.terms_fst_builder.insert_key(term.as_slice())
     }
 
     /// Finish the serialization for this term postings.
@@ -146,6 +147,9 @@ impl PostingsSerializer {
     /// using `VInt` encoding.
     pub fn close_term(&mut self) -> io::Result<()> {
         if self.term_open {
+
+            self.terms_fst_builder.insert_value(&self.current_term_info)?;
+
             if !self.doc_ids.is_empty() {
                 // we have doc ids waiting to be written
                 // this happens when the number of doc ids is
@@ -202,6 +206,7 @@ impl PostingsSerializer {
                      term_freq: u32,
                      position_deltas: &[u32])
                      -> io::Result<()> {
+        self.current_term_info.doc_freq += 1;
         self.doc_ids.push(doc_id);
         if self.text_indexing_options.is_termfreq_enabled() {
             self.term_freqs.push(term_freq as u32);

src/postings/term_info.rs

@@ -12,7 +12,7 @@ use std::io;
 /// * `postings_offset` : an offset in the `.idx` file 
 /// addressing the start of the posting list associated
 /// to this term.
-#[derive(Debug,Ord,PartialOrd,Eq,PartialEq,Clone)]
+#[derive(Debug,Default,Ord,PartialOrd,Eq,PartialEq,Clone)]
 pub struct TermInfo {
     /// Number of documents in the segment containing the term
     pub doc_freq: u32,

src/query/boolean_query/mod.rs

@@ -62,12 +62,14 @@ mod tests {
             }
             assert!(index_writer.commit().is_ok());
         }
+
         let make_term_query = |text: &str| {
             let term_query = TermQuery::new(Term::from_field_text(text_field, text), SegmentPostingsOption::NoFreq);
             let query: Box<Query> = box term_query;
             query
         };
 
+        index.load_searchers().unwrap();
 
         let matching_docs = |boolean_query: &Query| {
             let searcher = index.searcher();
@@ -100,8 +102,6 @@ mod tests {
         }
         {
             let boolean_query = BooleanQuery::from(vec![(Occur::MustNot, make_term_query("d")),]);
-            // TODO optimize this use case : only MustNot subqueries... no need
-            // to read any postings.
             assert_eq!(matching_docs(&boolean_query), Vec::new());
         }
     }

src/query/phrase_query/mod.rs

@@ -48,6 +48,7 @@ mod tests {
             assert!(index_writer.commit().is_ok());
         }
 
+        index.load_searchers().unwrap();
         let searcher = index.searcher();
         let test_query = |texts: Vec<&str>| {
             let mut test_collector = TestCollector::default();

src/schema/field_entry.rs

@@ -54,7 +54,7 @@ impl FieldEntry {
     pub fn is_indexed(&self,) -> bool {
         match self.field_type {
             FieldType::Str(ref options) => options.get_indexing_options().is_indexed(),
-            _ => false, // TODO handle u32 indexed
+            FieldType::U32(ref options) => options.is_indexed(),
         }
     }
     

src/schema/schema.rs

@@ -329,7 +329,6 @@ mod tests {
         schema_builder.add_u32_field("count", count_options);
         let schema = schema_builder.build();
         let schema_json: String = format!("{}", json::as_pretty_json(&schema));
-        println!("{}", schema_json);
         let expected = r#"[
   {
     "name": "title",
@@ -456,7 +455,6 @@ mod tests {
                 "author": "fulmicoton",
                 "count": -5
             }"#);
-            println!("{:?}", json_err);
             match json_err {
                 Err(DocParsingError::ValueError(_, ValueParsingError::TypeError(_))) => {
                     assert!(true);
@@ -472,7 +470,6 @@ mod tests {
                 "author": "fulmicoton",
                 "count": 5000000000
             }"#);
-            println!("{:?}", json_err);
             match json_err {
                 Err(DocParsingError::ValueError(_, ValueParsingError::OverflowError(_))) => {
                     assert!(true);

src/store/writer.rs

@@ -2,12 +2,9 @@ use directory::WritePtr;
 use DocId;
 use schema::FieldValue;
 use common::BinarySerializable;
-use std::io::Write;
-use std::io;
-use error::Result;
+use std::io::{self, Write};
 use lz4;
 use datastruct::SkipListBuilder;
-use super::StoreReader;
 
 const BLOCK_SIZE: usize = 16_384;
 
@@ -33,17 +30,6 @@ impl StoreWriter {
         }
     }
 
-    pub fn stack_reader(&mut self, reader: &StoreReader) -> Result<()> {
-        for doc_id in 0..reader.max_doc {
-            let doc = try!(reader.get(doc_id));
-            let field_values: Vec<&FieldValue> = doc.field_values()
-                .iter()
-                .collect();
-            try!(self.store(&field_values));
-        }
-        Ok(())
-    }
-
     pub fn store<'a>(&mut self, field_values: &[&'a FieldValue]) -> io::Result<()> {
         self.intermediary_buffer.clear();
         try!((field_values.len() as u32).serialize(&mut self.intermediary_buffer));