Making mmap a feature

Cargo.toml

@@ -16,7 +16,7 @@ byteorder = "1.0"
 lazy_static = "0.2.1"
 tinysegmenter = "0.1.0"
 regex = "0.2"
-fst = "0.2"
+fst = {version="0.2", default-features=false}
 atomicwrites = "0.1.3"
 tempfile = "2.1"
 log = "0.3.6"
@@ -58,10 +58,15 @@ debug-assertions = false
 
 
 [features]
-default = ["simdcompression"]
+default = ["mmap", "simdcompression"]
 simdcompression = []
 streamdict = []
+mmap = ["fst/mmap"]
 
 
 [badges]
 travis-ci = { repository = "tantivy-search/tantivy" }
+
+[[example]]
+name = "simple_search"
+required-features = ["mmap"]

src/compression/mod.rs

@@ -24,7 +24,7 @@ const MINI_BLOCK: usize = 1;
 
 /// Returns the size in bytes of a compressed block, given `num_bits`.
 pub fn compressed_block_size(num_bits: u8) -> usize {
-    1 + (num_bits as usize) * BitPackerImpl::BLOCK_LEN / 8
+    1 + (num_bits as usize) * COMPRESSION_BLOCK_SIZE / 8
 }
 
 pub struct BlockEncoder {

src/core/index.rs

@@ -6,7 +6,11 @@ use std::sync::Arc;
 use std::borrow::BorrowMut;
 use std::fmt;
 use core::SegmentId;
-use directory::{Directory, MmapDirectory, RAMDirectory};
+
+
+#[cfg(feature="mmap")]
+use directory::MmapDirectory;
+use directory::{Directory, RAMDirectory};
 use indexer::index_writer::open_index_writer;
 use core::searcher::Searcher;
 use std::convert::From;
@@ -61,6 +65,7 @@ impl Index {
     /// The index will use the `MMapDirectory`.
     ///
     /// If a previous index was in this directory, then its meta file will be destroyed.
+    #[cfg(feature="mmap")]
     pub fn create<P: AsRef<Path>>(directory_path: P, schema: Schema) -> Result<Index> {
         let mmap_directory = MmapDirectory::open(directory_path)?;
         let directory = ManagedDirectory::new(mmap_directory)?;
@@ -80,6 +85,7 @@ impl Index {
     ///
     /// The temp directory is only used for testing the `MmapDirectory`.
     /// For other unit tests, prefer the `RAMDirectory`, see: `create_in_ram`.
+    #[cfg(feature="mmap")]
     pub fn create_from_tempdir(schema: Schema) -> Result<Index> {
         let mmap_directory = MmapDirectory::create_from_tempdir()?;
         let directory = ManagedDirectory::new(mmap_directory)?;
@@ -107,6 +113,7 @@ impl Index {
     }
 
     /// Opens a new directory from an index path.
+    #[cfg(feature="mmap")]
     pub fn open<P: AsRef<Path>>(directory_path: P) -> Result<Index> {
         let mmap_directory = MmapDirectory::open(directory_path)?;
         let directory = ManagedDirectory::new(mmap_directory)?;

src/directory/managed_directory.rs

@@ -282,6 +282,7 @@ impl Clone for ManagedDirectory {
 mod tests {
 
     use super::*;
+    #[cfg(feature="mmap")]
     use directory::MmapDirectory;
     use std::path::Path;
     use std::io::Write;
@@ -293,6 +294,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(feature="mmap")]
     fn test_managed_directory() {
         let tempdir = TempDir::new("index").unwrap();
         let tempdir_path = PathBuf::from(tempdir.path());
@@ -341,6 +343,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(feature="mmap ")]
     fn test_managed_directory_gc_while_mmapped() {
         let tempdir = TempDir::new("index").unwrap();
         let tempdir_path = PathBuf::from(tempdir.path());
@@ -370,6 +373,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(feature="mmap")]
     fn test_managed_directory_protect() {
         let tempdir = TempDir::new("index").unwrap();
         let tempdir_path = PathBuf::from(tempdir.path());

src/directory/mod.rs

@@ -3,7 +3,10 @@
 WORM directory abstraction.
 
 */
+
+#[cfg(feature="mmap")]
 mod mmap_directory;
+
 mod ram_directory;
 mod directory;
 mod read_only_source;
@@ -18,6 +21,8 @@ use std::io::{BufWriter, Seek, Write};
 pub use self::read_only_source::ReadOnlySource;
 pub use self::directory::Directory;
 pub use self::ram_directory::RAMDirectory;
+
+#[cfg(feature="mmap")]
 pub use self::mmap_directory::MmapDirectory;
 
 pub(crate) use self::read_only_source::SourceRead;
@@ -51,6 +56,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(feature="mmap")]
     fn test_mmap_directory() {
         let mut mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
         test_directory(&mut mmap_directory);

src/directory/read_only_source.rs

@@ -1,3 +1,4 @@
+#[cfg(feature="mmap")]
 use fst::raw::MmapReadOnly;
 use std::ops::Deref;
 use super::shared_vec_slice::SharedVecSlice;
@@ -14,6 +15,7 @@ use stable_deref_trait::{CloneStableDeref, StableDeref};
 /// hold by this object should never be altered or destroyed.
 pub enum ReadOnlySource {
     /// Mmap source of data
+    #[cfg(feature="mmap")]
     Mmap(MmapReadOnly),
     /// Wrapping a `Vec<u8>`
     Anonymous(SharedVecSlice),
@@ -39,6 +41,7 @@ impl ReadOnlySource {
     /// Returns the data underlying the ReadOnlySource object.
     pub fn as_slice(&self) -> &[u8] {
         match *self {
+            #[cfg(feature="mmap")]
             ReadOnlySource::Mmap(ref mmap_read_only) => unsafe { mmap_read_only.as_slice() },
             ReadOnlySource::Anonymous(ref shared_vec) => shared_vec.as_slice(),
         }
@@ -65,6 +68,7 @@ impl ReadOnlySource {
     pub fn slice(&self, from_offset: usize, to_offset: usize) -> ReadOnlySource {
         assert!(from_offset <= to_offset, "Requested negative slice [{}..{}]", from_offset, to_offset);
         match *self {
+            #[cfg(feature="mmap")]
             ReadOnlySource::Mmap(ref mmap_read_only) => {
                 let sliced_mmap = mmap_read_only.range(from_offset, to_offset - from_offset);
                 ReadOnlySource::Mmap(sliced_mmap)

src/functional_test.rs

@@ -13,6 +13,7 @@ fn check_index_content(searcher: &Searcher, vals: &HashSet<u64>) {
 
 #[test]
 #[ignore]
+#[cfg(feature="mmap")]
 fn test_indexing() {
     let mut schema_builder = SchemaBuilder::default();
 

src/lib.rs

@@ -334,6 +334,7 @@ mod tests {
     }
 
     #[test]
+    #[cfg(feature="mmap")]
     fn test_indexing() {
         let mut schema_builder = SchemaBuilder::default();
         let text_field = schema_builder.add_text_field("text", TEXT);

src/query/boolean_query/mod.rs

@@ -25,7 +25,7 @@ mod tests {
         let mut schema_builder = SchemaBuilder::default();
         let text_field = schema_builder.add_text_field("text", TEXT);
         let schema = schema_builder.build();
-        let index = Index::create_from_tempdir(schema).unwrap();
+        let index = Index::create_in_ram(schema);
         {
             // writing the segment
             let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();

src/query/term_query/mod.rs

@@ -23,7 +23,7 @@ mod tests {
         let mut schema_builder = SchemaBuilder::default();
         let text_field = schema_builder.add_text_field("text", STRING);
         let schema = schema_builder.build();
-        let index = Index::create_from_tempdir(schema).unwrap();
+        let index = Index::create_in_ram(schema);
         {
             // writing the segment
             let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap();

src/store/mod.rs

@@ -48,7 +48,7 @@ mod tests {
     use schema::TextOptions;
     use schema::FieldValue;
     use schema::Document;
-    use directory::{Directory, MmapDirectory, RAMDirectory, WritePtr};
+    use directory::{Directory, RAMDirectory, WritePtr};
 
     fn write_lorem_ipsum_store(writer: WritePtr, num_docs: usize) -> Schema {
         let mut schema_builder = SchemaBuilder::default();
@@ -106,8 +106,9 @@ mod tests {
     }
 
     #[bench]
+    #[cfg(feature="mmap")]
     fn bench_store_encode(b: &mut Bencher) {
-        let mut directory = MmapDirectory::create_from_tempdir().unwrap();
+        let mut directory = RAMDirectory::create();
         let path = Path::new("store");
         b.iter(|| {
             write_lorem_ipsum_store(directory.open_write(path).unwrap(), 1_000);
@@ -117,7 +118,7 @@ mod tests {
 
     #[bench]
     fn bench_store_decode(b: &mut Bencher) {
-        let mut directory = MmapDirectory::create_from_tempdir().unwrap();
+        let mut directory = RAMDirectory::create();
         let path = Path::new("store");
         write_lorem_ipsum_store(directory.open_write(path).unwrap(), 1_000);
         let store_source = directory.open_read(path).unwrap();

src/termdict/fstdict/termdict.rs

@@ -86,6 +86,7 @@ fn open_fst_index(source: ReadOnlySource) -> fst::Map {
         ReadOnlySource::Anonymous(data) => {
             Fst::from_shared_bytes(data.data, data.start, data.len).expect("FST data is corrupted")
         }
+        #[cfg(feature="mmap")]
         ReadOnlySource::Mmap(mmap_readonly) => {
             Fst::from_mmap(mmap_readonly).expect("FST data is corrupted")
         }