diff --git a/Cargo.toml b/Cargo.toml index a4acb9c6b..724604681 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ byteorder = "1.0" lazy_static = "0.2.1" tinysegmenter = "0.1.0" regex = "0.2" -fst = "0.2" +fst = {version="0.2", default-features=false} atomicwrites = "0.1.3" tempfile = "2.1" log = "0.3.6" @@ -58,10 +58,15 @@ debug-assertions = false [features] -default = ["simdcompression"] +default = ["mmap", "simdcompression"] simdcompression = [] streamdict = [] +mmap = ["fst/mmap"] [badges] travis-ci = { repository = "tantivy-search/tantivy" } + +[[example]] +name = "simple_search" +required-features = ["mmap"] \ No newline at end of file diff --git a/src/compression/mod.rs b/src/compression/mod.rs index b593a513a..fea9587a1 100644 --- a/src/compression/mod.rs +++ b/src/compression/mod.rs @@ -24,7 +24,7 @@ const MINI_BLOCK: usize = 1; /// Returns the size in bytes of a compressed block, given `num_bits`. pub fn compressed_block_size(num_bits: u8) -> usize { - 1 + (num_bits as usize) * BitPackerImpl::BLOCK_LEN / 8 + 1 + (num_bits as usize) * COMPRESSION_BLOCK_SIZE / 8 } pub struct BlockEncoder { diff --git a/src/core/index.rs b/src/core/index.rs index 275ce721a..7312ea25d 100644 --- a/src/core/index.rs +++ b/src/core/index.rs @@ -6,7 +6,11 @@ use std::sync::Arc; use std::borrow::BorrowMut; use std::fmt; use core::SegmentId; -use directory::{Directory, MmapDirectory, RAMDirectory}; + + +#[cfg(feature="mmap")] +use directory::MmapDirectory; +use directory::{Directory, RAMDirectory}; use indexer::index_writer::open_index_writer; use core::searcher::Searcher; use std::convert::From; @@ -61,6 +65,7 @@ impl Index { /// The index will use the `MMapDirectory`. /// /// If a previous index was in this directory, then its meta file will be destroyed. + #[cfg(feature="mmap")] pub fn create>(directory_path: P, schema: Schema) -> Result { let mmap_directory = MmapDirectory::open(directory_path)?; let directory = ManagedDirectory::new(mmap_directory)?; @@ -80,6 +85,7 @@ impl Index { /// /// The temp directory is only used for testing the `MmapDirectory`. /// For other unit tests, prefer the `RAMDirectory`, see: `create_in_ram`. + #[cfg(feature="mmap")] pub fn create_from_tempdir(schema: Schema) -> Result { let mmap_directory = MmapDirectory::create_from_tempdir()?; let directory = ManagedDirectory::new(mmap_directory)?; @@ -107,6 +113,7 @@ impl Index { } /// Opens a new directory from an index path. + #[cfg(feature="mmap")] pub fn open>(directory_path: P) -> Result { let mmap_directory = MmapDirectory::open(directory_path)?; let directory = ManagedDirectory::new(mmap_directory)?; diff --git a/src/directory/managed_directory.rs b/src/directory/managed_directory.rs index 840568449..4d68dd221 100644 --- a/src/directory/managed_directory.rs +++ b/src/directory/managed_directory.rs @@ -282,6 +282,7 @@ impl Clone for ManagedDirectory { mod tests { use super::*; + #[cfg(feature="mmap")] use directory::MmapDirectory; use std::path::Path; use std::io::Write; @@ -293,6 +294,7 @@ mod tests { } #[test] + #[cfg(feature="mmap")] fn test_managed_directory() { let tempdir = TempDir::new("index").unwrap(); let tempdir_path = PathBuf::from(tempdir.path()); @@ -341,6 +343,7 @@ mod tests { } #[test] + #[cfg(feature="mmap ")] fn test_managed_directory_gc_while_mmapped() { let tempdir = TempDir::new("index").unwrap(); let tempdir_path = PathBuf::from(tempdir.path()); @@ -370,6 +373,7 @@ mod tests { } #[test] + #[cfg(feature="mmap")] fn test_managed_directory_protect() { let tempdir = TempDir::new("index").unwrap(); let tempdir_path = PathBuf::from(tempdir.path()); diff --git a/src/directory/mod.rs b/src/directory/mod.rs index 7d1c6fb17..e2a7d670e 100644 --- a/src/directory/mod.rs +++ b/src/directory/mod.rs @@ -3,7 +3,10 @@ WORM directory abstraction. */ + +#[cfg(feature="mmap")] mod mmap_directory; + mod ram_directory; mod directory; mod read_only_source; @@ -18,6 +21,8 @@ use std::io::{BufWriter, Seek, Write}; pub use self::read_only_source::ReadOnlySource; pub use self::directory::Directory; pub use self::ram_directory::RAMDirectory; + +#[cfg(feature="mmap")] pub use self::mmap_directory::MmapDirectory; pub(crate) use self::read_only_source::SourceRead; @@ -51,6 +56,7 @@ mod tests { } #[test] + #[cfg(feature="mmap")] fn test_mmap_directory() { let mut mmap_directory = MmapDirectory::create_from_tempdir().unwrap(); test_directory(&mut mmap_directory); diff --git a/src/directory/read_only_source.rs b/src/directory/read_only_source.rs index ec9e8aeb5..fe17742d7 100644 --- a/src/directory/read_only_source.rs +++ b/src/directory/read_only_source.rs @@ -1,3 +1,4 @@ +#[cfg(feature="mmap")] use fst::raw::MmapReadOnly; use std::ops::Deref; use super::shared_vec_slice::SharedVecSlice; @@ -14,6 +15,7 @@ use stable_deref_trait::{CloneStableDeref, StableDeref}; /// hold by this object should never be altered or destroyed. pub enum ReadOnlySource { /// Mmap source of data + #[cfg(feature="mmap")] Mmap(MmapReadOnly), /// Wrapping a `Vec` Anonymous(SharedVecSlice), @@ -39,6 +41,7 @@ impl ReadOnlySource { /// Returns the data underlying the ReadOnlySource object. pub fn as_slice(&self) -> &[u8] { match *self { + #[cfg(feature="mmap")] ReadOnlySource::Mmap(ref mmap_read_only) => unsafe { mmap_read_only.as_slice() }, ReadOnlySource::Anonymous(ref shared_vec) => shared_vec.as_slice(), } @@ -65,6 +68,7 @@ impl ReadOnlySource { pub fn slice(&self, from_offset: usize, to_offset: usize) -> ReadOnlySource { assert!(from_offset <= to_offset, "Requested negative slice [{}..{}]", from_offset, to_offset); match *self { + #[cfg(feature="mmap")] ReadOnlySource::Mmap(ref mmap_read_only) => { let sliced_mmap = mmap_read_only.range(from_offset, to_offset - from_offset); ReadOnlySource::Mmap(sliced_mmap) diff --git a/src/functional_test.rs b/src/functional_test.rs index d29f09ea6..a232e434d 100644 --- a/src/functional_test.rs +++ b/src/functional_test.rs @@ -13,6 +13,7 @@ fn check_index_content(searcher: &Searcher, vals: &HashSet) { #[test] #[ignore] +#[cfg(feature="mmap")] fn test_indexing() { let mut schema_builder = SchemaBuilder::default(); diff --git a/src/lib.rs b/src/lib.rs index 26b45d2d2..32bb51d5e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -334,6 +334,7 @@ mod tests { } #[test] + #[cfg(feature="mmap")] fn test_indexing() { let mut schema_builder = SchemaBuilder::default(); let text_field = schema_builder.add_text_field("text", TEXT); diff --git a/src/query/boolean_query/mod.rs b/src/query/boolean_query/mod.rs index 6df8b8cec..6ec61c56b 100644 --- a/src/query/boolean_query/mod.rs +++ b/src/query/boolean_query/mod.rs @@ -25,7 +25,7 @@ mod tests { let mut schema_builder = SchemaBuilder::default(); let text_field = schema_builder.add_text_field("text", TEXT); let schema = schema_builder.build(); - let index = Index::create_from_tempdir(schema).unwrap(); + let index = Index::create_in_ram(schema); { // writing the segment let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap(); diff --git a/src/query/term_query/mod.rs b/src/query/term_query/mod.rs index 455ec3ca5..77d4cfb29 100644 --- a/src/query/term_query/mod.rs +++ b/src/query/term_query/mod.rs @@ -23,7 +23,7 @@ mod tests { let mut schema_builder = SchemaBuilder::default(); let text_field = schema_builder.add_text_field("text", STRING); let schema = schema_builder.build(); - let index = Index::create_from_tempdir(schema).unwrap(); + let index = Index::create_in_ram(schema); { // writing the segment let mut index_writer = index.writer_with_num_threads(1, 40_000_000).unwrap(); diff --git a/src/store/mod.rs b/src/store/mod.rs index 15d4d51e1..152fe883a 100644 --- a/src/store/mod.rs +++ b/src/store/mod.rs @@ -48,7 +48,7 @@ mod tests { use schema::TextOptions; use schema::FieldValue; use schema::Document; - use directory::{Directory, MmapDirectory, RAMDirectory, WritePtr}; + use directory::{Directory, RAMDirectory, WritePtr}; fn write_lorem_ipsum_store(writer: WritePtr, num_docs: usize) -> Schema { let mut schema_builder = SchemaBuilder::default(); @@ -106,8 +106,9 @@ mod tests { } #[bench] + #[cfg(feature="mmap")] fn bench_store_encode(b: &mut Bencher) { - let mut directory = MmapDirectory::create_from_tempdir().unwrap(); + let mut directory = RAMDirectory::create(); let path = Path::new("store"); b.iter(|| { write_lorem_ipsum_store(directory.open_write(path).unwrap(), 1_000); @@ -117,7 +118,7 @@ mod tests { #[bench] fn bench_store_decode(b: &mut Bencher) { - let mut directory = MmapDirectory::create_from_tempdir().unwrap(); + let mut directory = RAMDirectory::create(); let path = Path::new("store"); write_lorem_ipsum_store(directory.open_write(path).unwrap(), 1_000); let store_source = directory.open_read(path).unwrap(); diff --git a/src/termdict/fstdict/termdict.rs b/src/termdict/fstdict/termdict.rs index 4b1d81c19..5f71b40fc 100644 --- a/src/termdict/fstdict/termdict.rs +++ b/src/termdict/fstdict/termdict.rs @@ -86,6 +86,7 @@ fn open_fst_index(source: ReadOnlySource) -> fst::Map { ReadOnlySource::Anonymous(data) => { Fst::from_shared_bytes(data.data, data.start, data.len).expect("FST data is corrupted") } + #[cfg(feature="mmap")] ReadOnlySource::Mmap(mmap_readonly) => { Fst::from_mmap(mmap_readonly).expect("FST data is corrupted") }