diff --git a/src/common/mod.rs b/src/common/mod.rs index 27228b057..6a1026e54 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -13,8 +13,6 @@ pub use self::serialize::{BinarySerializable, FixedSize}; pub use self::vint::{read_u32_vint, serialize_vint_u32, write_u32_vint, VInt}; pub use byteorder::LittleEndian as Endianness; -use std::io; - /// Computes the number of bits that will be used for bitpacking. /// /// In general the target is the minimum number of bits @@ -52,11 +50,6 @@ pub(crate) fn is_power_of_2(n: usize) -> bool { (n > 0) && (n & (n - 1) == 0) } -/// Create a default io error given a string. -pub(crate) fn make_io_err(msg: String) -> io::Error { - io::Error::new(io::ErrorKind::Other, msg) -} - /// Has length trait pub trait HasLen { /// Return length diff --git a/src/core/index.rs b/src/core/index.rs index 62a31cc95..5e8e08200 100644 --- a/src/core/index.rs +++ b/src/core/index.rs @@ -6,8 +6,6 @@ use core::SegmentId; use core::SegmentMeta; use core::META_FILEPATH; use directory::ManagedDirectory; -#[cfg(feature = "mmap")] -use directory::MmapDirectory; use directory::INDEX_WRITER_LOCK; use directory::{Directory, RAMDirectory}; use error::DataCorruption; @@ -24,12 +22,15 @@ use schema::Schema; use serde_json; use std::borrow::BorrowMut; use std::fmt; -use std::path::Path; use std::sync::Arc; use tokenizer::BoxedTokenizer; use tokenizer::TokenizerManager; use IndexWriter; use Result; +#[cfg(feature = "mmap")] +use std::path::Path; +#[cfg(feature = "mmap")] +use directory::MmapDirectory; fn load_metas(directory: &Directory) -> Result { let meta_data = directory.atomic_read(&META_FILEPATH)?; @@ -355,10 +356,8 @@ mod tests { use directory::RAMDirectory; use schema::Field; use schema::{Schema, INDEXED, TEXT}; - use std::path::PathBuf; use std::thread; use std::time::Duration; - use tempdir::TempDir; use Index; use IndexReader; use IndexWriter; @@ -444,62 +443,72 @@ mod tests { test_index_on_commit_reload_policy_aux(field, &mut writer, &reader); } - #[test] - fn test_index_on_commit_reload_policy_mmap() { - let schema = throw_away_schema(); - let field = schema.get_field("num_likes").unwrap(); - let tempdir = TempDir::new("index").unwrap(); - let tempdir_path = PathBuf::from(tempdir.path()); - let index = Index::create_in_dir(&tempdir_path, schema).unwrap(); - let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap(); - writer.commit().unwrap(); - let reader = index - .reader_builder() - .reload_policy(ReloadPolicy::OnCommit) - .try_into() - .unwrap(); - assert_eq!(reader.searcher().num_docs(), 0); - test_index_on_commit_reload_policy_aux(field, &mut writer, &reader); + + #[cfg(feature="mmap")] + mod mmap_specific { + + use std::path::PathBuf; + use tempdir::TempDir; + use super::*; + + #[test] + fn test_index_on_commit_reload_policy_mmap() { + let schema = throw_away_schema(); + let field = schema.get_field("num_likes").unwrap(); + let tempdir = TempDir::new("index").unwrap(); + let tempdir_path = PathBuf::from(tempdir.path()); + let index = Index::create_in_dir(&tempdir_path, schema).unwrap(); + let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap(); + writer.commit().unwrap(); + let reader = index + .reader_builder() + .reload_policy(ReloadPolicy::OnCommit) + .try_into() + .unwrap(); + assert_eq!(reader.searcher().num_docs(), 0); + test_index_on_commit_reload_policy_aux(field, &mut writer, &reader); + } + + #[test] + fn test_index_manual_policy_mmap() { + let schema = throw_away_schema(); + let field = schema.get_field("num_likes").unwrap(); + let index = Index::create_from_tempdir(schema).unwrap(); + let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap(); + writer.commit().unwrap(); + let reader = index + .reader_builder() + .reload_policy(ReloadPolicy::Manual) + .try_into() + .unwrap(); + assert_eq!(reader.searcher().num_docs(), 0); + writer.add_document(doc!(field=>1u64)); + writer.commit().unwrap(); + thread::sleep(Duration::from_millis(500)); + assert_eq!(reader.searcher().num_docs(), 0); + reader.reload().unwrap(); + assert_eq!(reader.searcher().num_docs(), 1); + } + + #[test] + fn test_index_on_commit_reload_policy_different_directories() { + let schema = throw_away_schema(); + let field = schema.get_field("num_likes").unwrap(); + let tempdir = TempDir::new("index").unwrap(); + let tempdir_path = PathBuf::from(tempdir.path()); + let write_index = Index::create_in_dir(&tempdir_path, schema).unwrap(); + let read_index = Index::open_in_dir(&tempdir_path).unwrap(); + let reader = read_index + .reader_builder() + .reload_policy(ReloadPolicy::OnCommit) + .try_into() + .unwrap(); + assert_eq!(reader.searcher().num_docs(), 0); + let mut writer = write_index.writer_with_num_threads(1, 3_000_000).unwrap(); + test_index_on_commit_reload_policy_aux(field, &mut writer, &reader); + } } - #[test] - fn test_index_manual_policy_mmap() { - let schema = throw_away_schema(); - let field = schema.get_field("num_likes").unwrap(); - let index = Index::create_from_tempdir(schema).unwrap(); - let mut writer = index.writer_with_num_threads(1, 3_000_000).unwrap(); - writer.commit().unwrap(); - let reader = index - .reader_builder() - .reload_policy(ReloadPolicy::Manual) - .try_into() - .unwrap(); - assert_eq!(reader.searcher().num_docs(), 0); - writer.add_document(doc!(field=>1u64)); - writer.commit().unwrap(); - thread::sleep(Duration::from_millis(500)); - assert_eq!(reader.searcher().num_docs(), 0); - reader.reload().unwrap(); - assert_eq!(reader.searcher().num_docs(), 1); - } - - #[test] - fn test_index_on_commit_reload_policy_different_directories() { - let schema = throw_away_schema(); - let field = schema.get_field("num_likes").unwrap(); - let tempdir = TempDir::new("index").unwrap(); - let tempdir_path = PathBuf::from(tempdir.path()); - let write_index = Index::create_in_dir(&tempdir_path, schema).unwrap(); - let read_index = Index::open_in_dir(&tempdir_path).unwrap(); - let reader = read_index - .reader_builder() - .reload_policy(ReloadPolicy::OnCommit) - .try_into() - .unwrap(); - assert_eq!(reader.searcher().num_docs(), 0); - let mut writer = write_index.writer_with_num_threads(1, 3_000_000).unwrap(); - test_index_on_commit_reload_policy_aux(field, &mut writer, &reader); - } fn test_index_on_commit_reload_policy_aux( field: Field, diff --git a/src/directory/managed_directory.rs b/src/directory/managed_directory.rs index ddd8dfbba..b2f2bfc34 100644 --- a/src/directory/managed_directory.rs +++ b/src/directory/managed_directory.rs @@ -260,95 +260,100 @@ impl Clone for ManagedDirectory { #[cfg(test)] mod tests { - use super::*; #[cfg(feature = "mmap")] - use directory::MmapDirectory; - use std::io::Write; - use std::path::Path; - use tempdir::TempDir; + mod mmap_specific { - lazy_static! { - static ref TEST_PATH1: &'static Path = Path::new("some_path_for_test"); - static ref TEST_PATH2: &'static Path = Path::new("some_path_for_test2"); - } + use super::super::*; + use std::path::Path; + use tempdir::TempDir; - #[test] - #[cfg(feature = "mmap")] - fn test_managed_directory() { - let tempdir = TempDir::new("index").unwrap(); - let tempdir_path = PathBuf::from(tempdir.path()); - { - let mmap_directory = MmapDirectory::open(&tempdir_path).unwrap(); - let mut managed_directory = ManagedDirectory::wrap(mmap_directory).unwrap(); + + lazy_static! { + static ref TEST_PATH1: &'static Path = Path::new("some_path_for_test"); + static ref TEST_PATH2: &'static Path = Path::new("some_path_for_test2"); + } + + use directory::MmapDirectory; + use std::io::Write; + + #[test] + fn test_managed_directory() { + let tempdir = TempDir::new("index").unwrap(); + let tempdir_path = PathBuf::from(tempdir.path()); { - let mut write_file = managed_directory.open_write(*TEST_PATH1).unwrap(); - write_file.flush().unwrap(); + let mmap_directory = MmapDirectory::open(&tempdir_path).unwrap(); + let mut managed_directory = ManagedDirectory::wrap(mmap_directory).unwrap(); + { + let mut write_file = managed_directory.open_write(*TEST_PATH1).unwrap(); + write_file.flush().unwrap(); + } + { + managed_directory + .atomic_write(*TEST_PATH2, &vec![0u8, 1u8]) + .unwrap(); + } + { + assert!(managed_directory.exists(*TEST_PATH1)); + assert!(managed_directory.exists(*TEST_PATH2)); + } + { + let living_files: HashSet = + [TEST_PATH1.to_owned()].into_iter().cloned().collect(); + managed_directory.garbage_collect(|| living_files); + } + { + assert!(managed_directory.exists(*TEST_PATH1)); + assert!(!managed_directory.exists(*TEST_PATH2)); + } } { - managed_directory - .atomic_write(*TEST_PATH2, &vec![0u8, 1u8]) - .unwrap(); - } - { - assert!(managed_directory.exists(*TEST_PATH1)); - assert!(managed_directory.exists(*TEST_PATH2)); - } - { - let living_files: HashSet = - [TEST_PATH1.to_owned()].into_iter().cloned().collect(); - managed_directory.garbage_collect(|| living_files); - } - { - assert!(managed_directory.exists(*TEST_PATH1)); - assert!(!managed_directory.exists(*TEST_PATH2)); + let mmap_directory = MmapDirectory::open(&tempdir_path).unwrap(); + let mut managed_directory = ManagedDirectory::wrap(mmap_directory).unwrap(); + { + assert!(managed_directory.exists(*TEST_PATH1)); + assert!(!managed_directory.exists(*TEST_PATH2)); + } + { + let living_files: HashSet = HashSet::new(); + managed_directory.garbage_collect(|| living_files); + } + { + assert!(!managed_directory.exists(*TEST_PATH1)); + assert!(!managed_directory.exists(*TEST_PATH2)); + } } } - { + + #[test] + fn test_managed_directory_gc_while_mmapped() { + let tempdir = TempDir::new("index").unwrap(); + let tempdir_path = PathBuf::from(tempdir.path()); + let living_files = HashSet::new(); + let mmap_directory = MmapDirectory::open(&tempdir_path).unwrap(); let mut managed_directory = ManagedDirectory::wrap(mmap_directory).unwrap(); - { - assert!(managed_directory.exists(*TEST_PATH1)); - assert!(!managed_directory.exists(*TEST_PATH2)); - } - { - let living_files: HashSet = HashSet::new(); - managed_directory.garbage_collect(|| living_files); - } - { - assert!(!managed_directory.exists(*TEST_PATH1)); - assert!(!managed_directory.exists(*TEST_PATH2)); - } - } - } - - #[test] - #[cfg(feature = "mmap ")] - fn test_managed_directory_gc_while_mmapped() { - let tempdir = TempDir::new("index").unwrap(); - let tempdir_path = PathBuf::from(tempdir.path()); - let living_files = HashSet::new(); - - let mmap_directory = MmapDirectory::open(&tempdir_path).unwrap(); - let mut managed_directory = ManagedDirectory::wrap(mmap_directory).unwrap(); - managed_directory - .atomic_write(*TEST_PATH1, &vec![0u8, 1u8]) - .unwrap(); - assert!(managed_directory.exists(*TEST_PATH1)); - - let _mmap_read = managed_directory.open_read(*TEST_PATH1).unwrap(); - managed_directory.garbage_collect(|| living_files.clone()); - if cfg!(target_os = "windows") { - // On Windows, gc should try and fail the file as it is mmapped. + managed_directory + .atomic_write(*TEST_PATH1, &vec![0u8, 1u8]) + .unwrap(); assert!(managed_directory.exists(*TEST_PATH1)); - // unmap should happen here. - drop(_mmap_read); - // The file should still be in the list of managed file and - // eventually be deleted once mmap is released. - managed_directory.garbage_collect(|| living_files); - assert!(!managed_directory.exists(*TEST_PATH1)); - } else { - assert!(!managed_directory.exists(*TEST_PATH1)); + + let _mmap_read = managed_directory.open_read(*TEST_PATH1).unwrap(); + managed_directory.garbage_collect(|| living_files.clone()); + if cfg!(target_os = "windows") { + // On Windows, gc should try and fail the file as it is mmapped. + assert!(managed_directory.exists(*TEST_PATH1)); + // unmap should happen here. + drop(_mmap_read); + // The file should still be in the list of managed file and + // eventually be deleted once mmap is released. + managed_directory.garbage_collect(|| living_files); + assert!(!managed_directory.exists(*TEST_PATH1)); + } else { + assert!(!managed_directory.exists(*TEST_PATH1)); + } } + + } } diff --git a/src/directory/mmap_directory.rs b/src/directory/mmap_directory.rs index 70c277d56..0f1f19384 100644 --- a/src/directory/mmap_directory.rs +++ b/src/directory/mmap_directory.rs @@ -6,7 +6,6 @@ use self::notify::RawEvent; use self::notify::RecursiveMode; use self::notify::Watcher; use atomicwrites; -use common::make_io_err; use core::META_FILEPATH; use directory::error::LockError; use directory::error::{DeleteError, IOError, OpenDirectoryError, OpenReadError, OpenWriteError}; @@ -37,6 +36,12 @@ use std::sync::Weak; use std::thread; use tempdir::TempDir; + +/// Create a default io error given a string. +pub(crate) fn make_io_err(msg: String) -> io::Error { + io::Error::new(io::ErrorKind::Other, msg) +} + /// Returns None iff the file exists, can be read, but is empty (and hence /// cannot be mmapped) fn open_mmap(full_path: &Path) -> result::Result, OpenReadError> { diff --git a/src/functional_test.rs b/src/functional_test.rs index ff36369ea..1c2a6501c 100644 --- a/src/functional_test.rs +++ b/src/functional_test.rs @@ -13,7 +13,6 @@ fn check_index_content(searcher: &Searcher, vals: &HashSet) { #[test] #[ignore] -#[cfg(feature = "mmap")] fn test_indexing() { let mut schema_builder = Schema::builder(); diff --git a/src/lib.rs b/src/lib.rs index 70c9e78fd..3a4a676dd 100755 --- a/src/lib.rs +++ b/src/lib.rs @@ -174,6 +174,7 @@ extern crate downcast_rs; #[macro_use] extern crate fail; +#[cfg(feature = "mmap")] #[cfg(test)] mod functional_test;