From 01b4aa9adc8596f54c745c20b428ac02b378380d Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Sun, 11 Oct 2020 22:22:56 +0900 Subject: [PATCH] Refactoring dir (#905) --- src/common/composite_file.rs | 2 +- src/core/index.rs | 5 +- src/directory/directory.rs | 4 +- src/directory/file_slice.rs | 185 +++++++++-------------------- src/directory/managed_directory.rs | 12 +- src/directory/mmap_directory.rs | 32 +++-- src/directory/owned_bytes.rs | 19 ++- src/directory/ram_directory.rs | 14 +-- src/directory/tests.rs | 72 +++++------ src/fastfield/delete.rs | 2 +- src/fastfield/mod.rs | 18 +-- src/fastfield/reader.rs | 2 +- src/indexer/segment_updater.rs | 4 +- src/positions/mod.rs | 2 +- src/postings/segment_postings.rs | 15 +-- src/store/mod.rs | 6 +- src/termdict/mod.rs | 16 +-- src/termdict/term_info_store.rs | 2 +- src/termdict/termdict.rs | 2 +- 19 files changed, 186 insertions(+), 228 deletions(-) diff --git a/src/common/composite_file.rs b/src/common/composite_file.rs index 775bd62a9..598274315 100644 --- a/src/common/composite_file.rs +++ b/src/common/composite_file.rs @@ -196,7 +196,7 @@ mod test { #[test] fn test_composite_file() -> crate::Result<()> { let path = Path::new("test_path"); - let mut directory = RAMDirectory::create(); + let directory = RAMDirectory::create(); { let w = directory.open_write(path).unwrap(); let mut composite_write = CompositeWrite::wrap(w); diff --git a/src/core/index.rs b/src/core/index.rs index 989b5766b..9781bef7d 100644 --- a/src/core/index.rs +++ b/src/core/index.rs @@ -21,7 +21,6 @@ use crate::schema::FieldType; use crate::schema::Schema; use crate::tokenizer::{TextAnalyzer, TokenizerManager}; use crate::IndexWriter; -use std::borrow::BorrowMut; use std::collections::HashSet; use std::fmt; @@ -153,8 +152,8 @@ impl Index { /// Create a new index from a directory. /// /// This will overwrite existing meta.json - fn from_directory(mut directory: ManagedDirectory, schema: Schema) -> crate::Result { - save_new_metas(schema.clone(), directory.borrow_mut())?; + fn from_directory(directory: ManagedDirectory, schema: Schema) -> crate::Result { + save_new_metas(schema.clone(), &directory)?; let metas = IndexMeta::with_schema(schema); Index::create_from_metas(directory, &metas, SegmentMetaInventory::default()) } diff --git a/src/directory/directory.rs b/src/directory/directory.rs index 024746dc3..5fbc141fb 100644 --- a/src/directory/directory.rs +++ b/src/directory/directory.rs @@ -152,7 +152,7 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static { /// was not called. /// /// The file may not previously exist. - fn open_write(&mut self, path: &Path) -> Result; + fn open_write(&self, path: &Path) -> Result; /// Reads the full content file that has been written using /// atomic_write. @@ -168,7 +168,7 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static { /// a partially written file. /// /// The file may or may not previously exist. - fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()>; + fn atomic_write(&self, path: &Path, data: &[u8]) -> io::Result<()>; /// Acquire a lock in the given directory. /// diff --git a/src/directory/file_slice.rs b/src/directory/file_slice.rs index f1ea23ecb..1bd01cb66 100644 --- a/src/directory/file_slice.rs +++ b/src/directory/file_slice.rs @@ -1,6 +1,7 @@ +use stable_deref_trait::StableDeref; + use crate::common::HasLen; use crate::directory::OwnedBytes; -use stable_deref_trait::{CloneStableDeref, StableDeref}; use std::sync::Arc; use std::{io, ops::Deref}; @@ -8,57 +9,81 @@ pub type BoxedData = Box + Send + Sync + 'static>; /// Objects that represents files sections in tantivy. /// -/// These read objects are only in charge to deliver -/// the data in the form of a constant read-only `&[u8]`. -/// Whatever happens to the directory file, the data -/// hold by this object should never be altered or destroyed. -pub trait FileSliceTrait: 'static + Send + Sync + HasLen { - fn read_bytes(&self) -> io::Result; - fn slice(&self, from: usize, to: usize) -> FileSlice; +/// By contract, whatever happens to the directory file, as long as a FileHandle +/// is alive, the data associated with it cannot be altered or destroyed. +/// +/// The underlying behavior is therefore specific to the `Directory` that created it. +/// Despite its name, a `FileSlice` may or may not directly map to an actual file +/// on the filesystem. +pub trait FileHandle: 'static + Send + Sync + HasLen { + fn read_bytes(&self, from: usize, to: usize) -> io::Result; } -impl FileSliceTrait for &'static [u8] { - fn read_bytes(&self) -> io::Result { - Ok(OwnedBytes::new(*self)) - } - - fn slice(&self, from: usize, to: usize) -> FileSlice { - FileSlice::from(&self[from..to]) +impl FileHandle for &'static [u8] { + fn read_bytes(&self, from: usize, to: usize) -> io::Result { + let bytes = &self[from..to]; + Ok(OwnedBytes::new(bytes)) } } -impl HasLen for &'static [u8] { +impl> HasLen for T { fn len(&self) -> usize { self.as_ref().len() } } +impl From for FileSlice +where + B: StableDeref + Deref + 'static + Send + Sync, +{ + fn from(bytes: B) -> FileSlice { + FileSlice::new(OwnedBytes::new(bytes)) + } +} + /// Logical slice of read only file in tantivy. // -/// In other words, it is more or less equivalent to the triplet `(file, start_byteoffset, stop_offset)`. +/// It can be cloned and sliced cheaply. /// -/// FileSlice is a simple wrapper over an `Arc>`. It can -/// be cloned cheaply. -/// -/// The underlying behavior is therefore specific to the `Directory` that created it. -/// Despite its name, a `FileSlice` may or may not directly map to an actual file -/// on the filesystem. #[derive(Clone)] -pub struct FileSlice(Arc>); +pub struct FileSlice { + data: Arc>, + start: usize, + stop: usize, +} impl FileSlice { - /// Creates a FileSlice, wrapping over a FileSliceTrait. + /// Wraps a new `Deref` pub fn new(data: D) -> Self where - D: Deref + Send + Sync + 'static, + D: FileHandle, { - FileSlice::from(SlicedDeref::new(data)) + let len = data.len(); + FileSlice { + data: Arc::new(Box::new(data)), + start: 0, + stop: len, + } + } + + /// Creates a fileslice that is just a view over a slice of the data. + /// + /// # Panics + /// Panics if `to < from` or if `to` exceeds the filesize. + pub fn slice(&self, from: usize, to: usize) -> FileSlice { + assert!(to <= self.len()); + assert!(to >= from); + FileSlice { + data: self.data.clone(), + start: self.start + from, + stop: self.start + to, + } } /// Creates an empty FileSlice pub fn empty() -> FileSlice { - let data: &'static [u8] = &[]; - FileSlice::from(data) + const EMPTY_SLICE: &'static [u8] = &[]; + FileSlice::from(EMPTY_SLICE) } /// Returns a `OwnedBytes` with all of the data in the `FileSlice`. @@ -68,10 +93,10 @@ impl FileSlice { /// In particular, it is up to the `Directory` implementation /// to handle caching if needed. pub fn read_bytes(&self) -> io::Result { - self.0.read_bytes() + self.data.read_bytes(self.start, self.stop) } - /// Splits the file slice at the given offset and return two file slices. + /// Splits the FileSlice at the given offset and return two file slices. /// `file_slice[..split_offset]` and `file_slice[split_offset..]`. /// /// This operation is cheap and must not copy any underlying data. @@ -88,18 +113,6 @@ impl FileSlice { self.split(left_len) } - /// Creates a FileSlice that is just a view over a slice of the data. - pub fn slice(&self, start: usize, stop: usize) -> FileSlice { - assert!( - start <= stop, - "Requested negative slice [{}..{}]", - start, - stop - ); - assert!(stop <= self.len()); - self.0.slice(start, stop) - } - /// Like `.slice(...)` but enforcing only the `from` /// boundary. /// @@ -118,94 +131,14 @@ impl FileSlice { } impl HasLen for FileSlice { - fn len(&self) -> usize { - self.0.len() - } -} - -impl From for FileSlice { - fn from(file: S) -> Self { - FileSlice(Arc::new(Box::new(file))) - } -} - -impl From> for FileSlice { - fn from(data: Arc) -> Self { - let slice_deref: SlicedDeref = SlicedDeref::from(data); - FileSlice::from(slice_deref) - } -} - -/// `SliceDeref` wraps an `Arc` to implement `FileSliceTrait` . -/// It keeps track of (start, stop) boundaries. -#[derive(Clone)] -pub struct SlicedDeref { - data: Arc, - start: usize, - stop: usize, -} - -impl SlicedDeref { - /// Wraps a new `Deref` - pub fn new(data: D) -> Self - where - D: Deref + 'static + Send + Sync, - { - let len = data.len(); - SlicedDeref { - data: Arc::new(Box::new(data)), - start: 0, - stop: len, - } - } -} - -impl From> for SlicedDeref { - fn from(data: Arc) -> Self { - let len = data.len(); - SlicedDeref { - data, - start: 0, - stop: len, - } - } -} - -unsafe impl StableDeref for SlicedDeref {} -unsafe impl CloneStableDeref for SlicedDeref {} - -impl FileSliceTrait for SlicedDeref { - fn read_bytes(&self) -> io::Result { - Ok(OwnedBytes::new(self.clone())) - } - - fn slice(&self, from: usize, to: usize) -> FileSlice { - assert!(to <= self.len()); - FileSlice::from(SlicedDeref { - data: self.data.clone(), - start: self.start + from, - stop: self.start + to, - }) - } -} - -impl HasLen for SlicedDeref { fn len(&self) -> usize { self.stop - self.start } } -impl Deref for SlicedDeref { - type Target = [u8]; - - fn deref(&self) -> &Self::Target { - &self.data.deref()[self.start..self.stop] - } -} - #[cfg(test)] mod tests { - use super::{FileSlice, FileSliceTrait, SlicedDeref}; + use super::{FileHandle, FileSlice}; use crate::common::HasLen; use std::io; @@ -249,13 +182,13 @@ mod tests { #[test] fn test_file_slice_trait_slice_len() { let blop: &'static [u8] = b"abc"; - let owned_bytes: Box = Box::new(blop); + let owned_bytes: Box = Box::new(blop); assert_eq!(owned_bytes.len(), 3); } #[test] fn test_slice_deref() -> io::Result<()> { - let slice_deref = SlicedDeref::new(&b"abcdef"[..]); + let slice_deref = FileSlice::new(&b"abcdef"[..]); assert_eq!(slice_deref.len(), 6); assert_eq!(slice_deref.read_bytes()?.as_ref(), b"abcdef"); assert_eq!(slice_deref.slice(1, 4).read_bytes()?.as_ref(), b"bcd"); diff --git a/src/directory/managed_directory.rs b/src/directory/managed_directory.rs index 13c6cf762..fa3ce79cc 100644 --- a/src/directory/managed_directory.rs +++ b/src/directory/managed_directory.rs @@ -53,7 +53,7 @@ struct MetaInformation { /// Saves the file containing the list of existing files /// that were created by tantivy. fn save_managed_paths( - directory: &mut dyn Directory, + directory: &dyn Directory, wlock: &RwLockWriteGuard<'_, MetaInformation>, ) -> io::Result<()> { let mut w = serde_json::to_vec(&wlock.managed_paths)?; @@ -212,7 +212,7 @@ impl ManagedDirectory { /// File starting by "." are reserved to locks. /// They are not managed and cannot be subjected /// to garbage collection. - fn register_file_as_managed(&mut self, filepath: &Path) -> io::Result<()> { + fn register_file_as_managed(&self, filepath: &Path) -> io::Result<()> { // Files starting by "." (e.g. lock files) are not managed. if !is_managed(filepath) { return Ok(()); @@ -223,7 +223,7 @@ impl ManagedDirectory { .expect("Managed file lock poisoned"); let has_changed = meta_wlock.managed_paths.insert(filepath.to_owned()); if has_changed { - save_managed_paths(self.directory.as_mut(), &meta_wlock)?; + save_managed_paths(self.directory.as_ref(), &meta_wlock)?; } Ok(()) } @@ -282,7 +282,7 @@ impl Directory for ManagedDirectory { Ok(reader) } - fn open_write(&mut self, path: &Path) -> result::Result { + fn open_write(&self, path: &Path) -> result::Result { self.register_file_as_managed(path) .map_err(|io_error| OpenWriteError::wrap_io_error(io_error, path.to_path_buf()))?; Ok(io::BufWriter::new(Box::new(FooterProxy::new( @@ -294,7 +294,7 @@ impl Directory for ManagedDirectory { )))) } - fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()> { + fn atomic_write(&self, path: &Path, data: &[u8]) -> io::Result<()> { self.register_file_as_managed(path)?; self.directory.atomic_write(path, data) } @@ -416,7 +416,7 @@ mod tests_mmap_specific { let tempdir_path = PathBuf::from(tempdir.path()); let mmap_directory = MmapDirectory::open(&tempdir_path)?; - let mut managed_directory = ManagedDirectory::wrap(mmap_directory)?; + let managed_directory = ManagedDirectory::wrap(mmap_directory)?; let mut write = managed_directory.open_write(test_path1)?; write.write_all(&[0u8, 1u8])?; write.terminate()?; diff --git a/src/directory/mmap_directory.rs b/src/directory/mmap_directory.rs index be6942615..89bdffff8 100644 --- a/src/directory/mmap_directory.rs +++ b/src/directory/mmap_directory.rs @@ -17,7 +17,7 @@ use notify::RawEvent; use notify::RecursiveMode; use notify::Watcher; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use stable_deref_trait::StableDeref; use std::convert::From; use std::fmt; use std::fs::OpenOptions; @@ -32,6 +32,7 @@ use std::sync::Mutex; use std::sync::RwLock; use std::sync::Weak; use std::thread; +use std::{collections::HashMap, ops::Deref}; use tempfile::TempDir; /// Create a default io error given a string. @@ -400,6 +401,18 @@ impl TerminatingWrite for SafeFileWriter { } } +#[derive(Clone)] +struct MmapArc(Arc + Send + Sync>>); + +impl Deref for MmapArc { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + self.0.deref() + } +} +unsafe impl StableDeref for MmapArc {} + impl Directory for MmapDirectory { fn open_read(&self, path: &Path) -> result::Result { debug!("Open Read {:?}", path); @@ -414,10 +427,11 @@ impl Directory for MmapDirectory { let io_err = make_io_err(msg); OpenReadError::wrap_io_error(io_err, path.to_path_buf()) })?; - Ok(mmap_cache - .get_mmap(&full_path)? - .map(FileSlice::from) - .unwrap_or_else(FileSlice::empty)) + if let Some(mmap_arc) = mmap_cache.get_mmap(&full_path)? { + Ok(FileSlice::from(MmapArc(mmap_arc))) + } else { + Ok(FileSlice::empty()) + } } /// Any entry associated to the path in the mmap will be @@ -447,7 +461,7 @@ impl Directory for MmapDirectory { full_path.exists() } - fn open_write(&mut self, path: &Path) -> Result { + fn open_write(&self, path: &Path) -> Result { debug!("Open Write {:?}", path); let full_path = self.resolve_path(path); @@ -497,7 +511,7 @@ impl Directory for MmapDirectory { } } - fn atomic_write(&mut self, path: &Path, content: &[u8]) -> io::Result<()> { + fn atomic_write(&self, path: &Path, content: &[u8]) -> io::Result<()> { debug!("Atomic Write {:?}", path); let mut tempfile = tempfile::Builder::new().tempfile_in(&self.inner.root_path)?; tempfile.write_all(content)?; @@ -557,7 +571,7 @@ mod tests { // cannot be mmapped. // // In that case the directory returns a SharedVecSlice. - let mut mmap_directory = MmapDirectory::create_from_tempdir().unwrap(); + let mmap_directory = MmapDirectory::create_from_tempdir().unwrap(); let path = PathBuf::from("test"); { let mut w = mmap_directory.open_write(&path).unwrap(); @@ -573,7 +587,7 @@ mod tests { // here we test if the cache releases // mmaps correctly. - let mut mmap_directory = MmapDirectory::create_from_tempdir().unwrap(); + let mmap_directory = MmapDirectory::create_from_tempdir().unwrap(); let num_paths = 10; let paths: Vec = (0..num_paths) .map(|i| PathBuf::from(&*format!("file_{}", i))) diff --git a/src/directory/owned_bytes.rs b/src/directory/owned_bytes.rs index 7aa366ceb..89ec8d74e 100644 --- a/src/directory/owned_bytes.rs +++ b/src/directory/owned_bytes.rs @@ -4,6 +4,8 @@ use std::ops::Deref; use std::sync::Arc; use std::{fmt, io}; +use super::file_slice::FileHandle; + /// An OwnedBytes simply wraps an object that owns a slice of data and exposes /// this data as a static slice. /// @@ -14,6 +16,12 @@ pub struct OwnedBytes { box_stable_deref: Arc + Sync + Send>, } +impl FileHandle for OwnedBytes { + fn read_bytes(&self, from: usize, to: usize) -> io::Result { + Ok(self.slice(from, to)) + } +} + impl OwnedBytes { /// Creates an empty `OwnedBytes`. pub fn empty() -> OwnedBytes { @@ -25,13 +33,22 @@ impl OwnedBytes { data_holder: T, ) -> OwnedBytes { let box_stable_deref = Arc::new(data_holder); - let data = unsafe { mem::transmute::<_, &'static [u8]>(box_stable_deref.deref().deref()) }; + let bytes: &[u8] = box_stable_deref.as_ref(); + let data = unsafe { mem::transmute::<_, &'static [u8]>(bytes.deref()) }; OwnedBytes { box_stable_deref, data, } } + /// creates a fileslice that is just a view over a slice of the data. + pub fn slice(&self, from: usize, to: usize) -> Self { + OwnedBytes { + data: &self.data[from..to], + box_stable_deref: self.box_stable_deref.clone(), + } + } + /// Returns the underlying slice of data. /// `Deref` and `AsRef` are also available. #[inline(always)] diff --git a/src/directory/ram_directory.rs b/src/directory/ram_directory.rs index 0dc7dc572..3a8450307 100644 --- a/src/directory/ram_directory.rs +++ b/src/directory/ram_directory.rs @@ -86,7 +86,7 @@ struct InnerDirectory { impl InnerDirectory { fn write(&mut self, path: PathBuf, data: &[u8]) -> bool { - let data = FileSlice::new(Vec::from(data)); + let data = FileSlice::from(data.to_vec()); self.fs.insert(path, data).is_some() } @@ -151,7 +151,7 @@ impl RAMDirectory { /// written using the `atomic_write` api. /// /// If an error is encounterred, files may be persisted partially. - pub fn persist(&self, dest: &mut dyn Directory) -> crate::Result<()> { + pub fn persist(&self, dest: &dyn Directory) -> crate::Result<()> { let wlock = self.fs.write().unwrap(); for (path, file) in wlock.fs.iter() { let mut dest_wrt = dest.open_write(path)?; @@ -181,7 +181,7 @@ impl Directory for RAMDirectory { self.fs.read().unwrap().exists(path) } - fn open_write(&mut self, path: &Path) -> Result { + fn open_write(&self, path: &Path) -> Result { let mut fs = self.fs.write().unwrap(); let path_buf = PathBuf::from(path); let vec_writer = VecWriter::new(path_buf.clone(), self.clone()); @@ -205,7 +205,7 @@ impl Directory for RAMDirectory { Ok(bytes.as_slice().to_owned()) } - fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()> { + fn atomic_write(&self, path: &Path, data: &[u8]) -> io::Result<()> { fail_point!("RAMDirectory::atomic_write", |msg| Err(io::Error::new( io::ErrorKind::Other, msg.unwrap_or_else(|| "Undefined".to_string()) @@ -242,13 +242,13 @@ mod tests { let msg_seq: &'static [u8] = b"sequential is the way"; let path_atomic: &'static Path = Path::new("atomic"); let path_seq: &'static Path = Path::new("seq"); - let mut directory = RAMDirectory::create(); + let directory = RAMDirectory::create(); assert!(directory.atomic_write(path_atomic, msg_atomic).is_ok()); let mut wrt = directory.open_write(path_seq).unwrap(); assert!(wrt.write_all(msg_seq).is_ok()); assert!(wrt.flush().is_ok()); - let mut directory_copy = RAMDirectory::create(); - assert!(directory.persist(&mut directory_copy).is_ok()); + let directory_copy = RAMDirectory::create(); + assert!(directory.persist(&directory_copy).is_ok()); assert_eq!(directory_copy.atomic_read(path_atomic).unwrap(), msg_atomic); assert_eq!(directory_copy.atomic_read(path_seq).unwrap(), msg_seq); } diff --git a/src/directory/tests.rs b/src/directory/tests.rs index 5a5e4e9e3..dceda43f2 100644 --- a/src/directory/tests.rs +++ b/src/directory/tests.rs @@ -21,46 +21,46 @@ mod mmap_directory_tests { #[test] fn test_simple() -> crate::Result<()> { - let mut directory = make_directory(); - super::test_simple(&mut directory) + let directory = make_directory(); + super::test_simple(&directory) } #[test] fn test_write_create_the_file() { - let mut directory = make_directory(); - super::test_write_create_the_file(&mut directory); + let directory = make_directory(); + super::test_write_create_the_file(&directory); } #[test] fn test_rewrite_forbidden() -> crate::Result<()> { - let mut directory = make_directory(); - super::test_rewrite_forbidden(&mut directory)?; + let directory = make_directory(); + super::test_rewrite_forbidden(&directory)?; Ok(()) } #[test] fn test_directory_delete() -> crate::Result<()> { - let mut directory = make_directory(); - super::test_directory_delete(&mut directory)?; + let directory = make_directory(); + super::test_directory_delete(&directory)?; Ok(()) } #[test] fn test_lock_non_blocking() { - let mut directory = make_directory(); - super::test_lock_non_blocking(&mut directory); + let directory = make_directory(); + super::test_lock_non_blocking(&directory); } #[test] fn test_lock_blocking() { - let mut directory = make_directory(); - super::test_lock_blocking(&mut directory); + let directory = make_directory(); + super::test_lock_blocking(&directory); } #[test] fn test_watch() { - let mut directory = make_directory(); - super::test_watch(&mut directory); + let directory = make_directory(); + super::test_watch(&directory); } } @@ -75,46 +75,46 @@ mod ram_directory_tests { #[test] fn test_simple() -> crate::Result<()> { - let mut directory = make_directory(); - super::test_simple(&mut directory) + let directory = make_directory(); + super::test_simple(&directory) } #[test] fn test_write_create_the_file() { - let mut directory = make_directory(); - super::test_write_create_the_file(&mut directory); + let directory = make_directory(); + super::test_write_create_the_file(&directory); } #[test] fn test_rewrite_forbidden() -> crate::Result<()> { - let mut directory = make_directory(); - super::test_rewrite_forbidden(&mut directory)?; + let directory = make_directory(); + super::test_rewrite_forbidden(&directory)?; Ok(()) } #[test] fn test_directory_delete() -> crate::Result<()> { - let mut directory = make_directory(); - super::test_directory_delete(&mut directory)?; + let directory = make_directory(); + super::test_directory_delete(&directory)?; Ok(()) } #[test] fn test_lock_non_blocking() { - let mut directory = make_directory(); - super::test_lock_non_blocking(&mut directory); + let directory = make_directory(); + super::test_lock_non_blocking(&directory); } #[test] fn test_lock_blocking() { - let mut directory = make_directory(); - super::test_lock_blocking(&mut directory); + let directory = make_directory(); + super::test_lock_blocking(&directory); } #[test] fn test_watch() { - let mut directory = make_directory(); - super::test_watch(&mut directory); + let directory = make_directory(); + super::test_watch(&directory); } } @@ -122,12 +122,12 @@ mod ram_directory_tests { #[should_panic] fn ram_directory_panics_if_flush_forgotten() { let test_path: &'static Path = Path::new("some_path_for_test"); - let mut ram_directory = RAMDirectory::create(); + let ram_directory = RAMDirectory::create(); let mut write_file = ram_directory.open_write(test_path).unwrap(); assert!(write_file.write_all(&[4]).is_ok()); } -fn test_simple(directory: &mut dyn Directory) -> crate::Result<()> { +fn test_simple(directory: &dyn Directory) -> crate::Result<()> { let test_path: &'static Path = Path::new("some_path_for_test"); let mut write_file = directory.open_write(test_path)?; assert!(directory.exists(test_path)); @@ -143,7 +143,7 @@ fn test_simple(directory: &mut dyn Directory) -> crate::Result<()> { Ok(()) } -fn test_rewrite_forbidden(directory: &mut dyn Directory) -> crate::Result<()> { +fn test_rewrite_forbidden(directory: &dyn Directory) -> crate::Result<()> { let test_path: &'static Path = Path::new("some_path_for_test"); directory.open_write(test_path)?; assert!(directory.exists(test_path)); @@ -152,7 +152,7 @@ fn test_rewrite_forbidden(directory: &mut dyn Directory) -> crate::Result<()> { Ok(()) } -fn test_write_create_the_file(directory: &mut dyn Directory) { +fn test_write_create_the_file(directory: &dyn Directory) { let test_path: &'static Path = Path::new("some_path_for_test"); { assert!(directory.open_read(test_path).is_err()); @@ -163,7 +163,7 @@ fn test_write_create_the_file(directory: &mut dyn Directory) { } } -fn test_directory_delete(directory: &mut dyn Directory) -> crate::Result<()> { +fn test_directory_delete(directory: &dyn Directory) -> crate::Result<()> { let test_path: &'static Path = Path::new("some_path_for_test"); assert!(directory.open_read(test_path).is_err()); let mut write_file = directory.open_write(&test_path)?; @@ -189,7 +189,7 @@ fn test_directory_delete(directory: &mut dyn Directory) -> crate::Result<()> { Ok(()) } -fn test_watch(directory: &mut dyn Directory) { +fn test_watch(directory: &dyn Directory) { let num_progress: Arc = Default::default(); let counter: Arc = Default::default(); let counter_clone = counter.clone(); @@ -224,7 +224,7 @@ fn test_watch(directory: &mut dyn Directory) { assert!(10 <= counter.load(SeqCst)); } -fn test_lock_non_blocking(directory: &mut dyn Directory) { +fn test_lock_non_blocking(directory: &dyn Directory) { { let lock_a_res = directory.acquire_lock(&Lock { filepath: PathBuf::from("a.lock"), @@ -249,7 +249,7 @@ fn test_lock_non_blocking(directory: &mut dyn Directory) { assert!(lock_a_res.is_ok()); } -fn test_lock_blocking(directory: &mut dyn Directory) { +fn test_lock_blocking(directory: &dyn Directory) { let lock_a_res = directory.acquire_lock(&Lock { filepath: PathBuf::from("a.lock"), is_blocking: true, diff --git a/src/fastfield/delete.rs b/src/fastfield/delete.rs index b726a0e3a..58d9b77b3 100644 --- a/src/fastfield/delete.rs +++ b/src/fastfield/delete.rs @@ -54,7 +54,7 @@ impl DeleteBitSet { for &doc in docs { bitset.insert(doc); } - let mut directory = RAMDirectory::create(); + let directory = RAMDirectory::create(); let path = Path::new("dummydeletebitset"); let mut wrt = directory.open_write(path).unwrap(); write_delete_bitset(&bitset, max_doc, &mut wrt).unwrap(); diff --git a/src/fastfield/mod.rs b/src/fastfield/mod.rs index 227cc1e2f..dbccd900d 100644 --- a/src/fastfield/mod.rs +++ b/src/fastfield/mod.rs @@ -242,7 +242,7 @@ mod tests { #[test] fn test_intfastfield_small() -> crate::Result<()> { let path = Path::new("test"); - let mut directory: RAMDirectory = RAMDirectory::create(); + let directory: RAMDirectory = RAMDirectory::create(); { let write: WritePtr = directory.open_write(Path::new("test")).unwrap(); let mut serializer = FastFieldSerializer::from_write(write).unwrap(); @@ -269,7 +269,7 @@ mod tests { #[test] fn test_intfastfield_large() -> crate::Result<()> { let path = Path::new("test"); - let mut directory: RAMDirectory = RAMDirectory::create(); + let directory: RAMDirectory = RAMDirectory::create(); { let write: WritePtr = directory.open_write(Path::new("test"))?; let mut serializer = FastFieldSerializer::from_write(write)?; @@ -308,7 +308,7 @@ mod tests { #[test] fn test_intfastfield_null_amplitude() -> crate::Result<()> { let path = Path::new("test"); - let mut directory: RAMDirectory = RAMDirectory::create(); + let directory: RAMDirectory = RAMDirectory::create(); { let write: WritePtr = directory.open_write(Path::new("test")).unwrap(); @@ -338,7 +338,7 @@ mod tests { #[test] fn test_intfastfield_large_numbers() -> crate::Result<()> { let path = Path::new("test"); - let mut directory: RAMDirectory = RAMDirectory::create(); + let directory: RAMDirectory = RAMDirectory::create(); { let write: WritePtr = directory.open_write(Path::new("test")).unwrap(); @@ -374,7 +374,7 @@ mod tests { #[test] fn test_signed_intfastfield() -> crate::Result<()> { let path = Path::new("test"); - let mut directory: RAMDirectory = RAMDirectory::create(); + let directory: RAMDirectory = RAMDirectory::create(); let mut schema_builder = Schema::builder(); let i64_field = schema_builder.add_i64_field("field", FAST); @@ -417,7 +417,7 @@ mod tests { #[test] fn test_signed_intfastfield_default_val() -> crate::Result<()> { let path = Path::new("test"); - let mut directory: RAMDirectory = RAMDirectory::create(); + let directory: RAMDirectory = RAMDirectory::create(); let mut schema_builder = Schema::builder(); let i64_field = schema_builder.add_i64_field("field", FAST); let schema = schema_builder.build(); @@ -456,7 +456,7 @@ mod tests { let path = Path::new("test"); let permutation = generate_permutation(); let n = permutation.len(); - let mut directory = RAMDirectory::create(); + let directory = RAMDirectory::create(); { let write: WritePtr = directory.open_write(Path::new("test"))?; let mut serializer = FastFieldSerializer::from_write(write)?; @@ -612,7 +612,7 @@ mod bench { fn bench_intfastfield_linear_fflookup(b: &mut Bencher) { let path = Path::new("test"); let permutation = generate_permutation(); - let mut directory: RAMDirectory = RAMDirectory::create(); + let directory: RAMDirectory = RAMDirectory::create(); { let write: WritePtr = directory.open_write(Path::new("test")).unwrap(); let mut serializer = FastFieldSerializer::from_write(write).unwrap(); @@ -646,7 +646,7 @@ mod bench { fn bench_intfastfield_fflookup(b: &mut Bencher) { let path = Path::new("test"); let permutation = generate_permutation(); - let mut directory: RAMDirectory = RAMDirectory::create(); + let directory: RAMDirectory = RAMDirectory::create(); { let write: WritePtr = directory.open_write(Path::new("test")).unwrap(); let mut serializer = FastFieldSerializer::from_write(write).unwrap(); diff --git a/src/fastfield/reader.rs b/src/fastfield/reader.rs index ab5fa88ba..37a7e449a 100644 --- a/src/fastfield/reader.rs +++ b/src/fastfield/reader.rs @@ -127,7 +127,7 @@ impl From> for FastFieldReader { let field = schema_builder.add_u64_field("field", FAST); let schema = schema_builder.build(); let path = Path::new("__dummy__"); - let mut directory: RAMDirectory = RAMDirectory::create(); + let directory: RAMDirectory = RAMDirectory::create(); { let write: WritePtr = directory .open_write(path) diff --git a/src/indexer/segment_updater.rs b/src/indexer/segment_updater.rs index 58a38829e..159aee91c 100644 --- a/src/indexer/segment_updater.rs +++ b/src/indexer/segment_updater.rs @@ -43,7 +43,7 @@ const NUM_MERGE_THREADS: usize = 4; /// and flushed. /// /// This method is not part of tantivy's public API -pub fn save_new_metas(schema: Schema, directory: &mut dyn Directory) -> crate::Result<()> { +pub fn save_new_metas(schema: Schema, directory: &dyn Directory) -> crate::Result<()> { save_metas( &IndexMeta { segments: Vec::new(), @@ -64,7 +64,7 @@ pub fn save_new_metas(schema: Schema, directory: &mut dyn Directory) -> crate::R /// and flushed. /// /// This method is not part of tantivy's public API -fn save_metas(metas: &IndexMeta, directory: &mut dyn Directory) -> crate::Result<()> { +fn save_metas(metas: &IndexMeta, directory: &dyn Directory) -> crate::Result<()> { info!("save metas"); let mut buffer = serde_json::to_vec_pretty(metas)?; // Just adding a new line at the end of the buffer. diff --git a/src/positions/mod.rs b/src/positions/mod.rs index 8532b343d..d2bc27855 100644 --- a/src/positions/mod.rs +++ b/src/positions/mod.rs @@ -53,7 +53,7 @@ pub mod tests { } serializer.close().unwrap(); } - (FileSlice::new(stream_buffer), FileSlice::new(skip_buffer)) + (FileSlice::from(stream_buffer), FileSlice::from(skip_buffer)) } #[test] diff --git a/src/postings/segment_postings.rs b/src/postings/segment_postings.rs index 89aa04bb6..003d6ffa2 100644 --- a/src/postings/segment_postings.rs +++ b/src/postings/segment_postings.rs @@ -1,21 +1,16 @@ use crate::common::HasLen; - +use crate::directory::FileSlice; use crate::docset::DocSet; +use crate::fastfield::DeleteBitSet; use crate::positions::PositionReader; - use crate::postings::compression::COMPRESSION_BLOCK_SIZE; use crate::postings::serializer::PostingsSerializer; use crate::postings::BlockSearcher; - +use crate::postings::BlockSegmentPostings; use crate::postings::Postings; - use crate::schema::IndexRecordOption; use crate::{DocId, TERMINATED}; -use crate::directory::FileSlice; -use crate::fastfield::DeleteBitSet; -use crate::postings::BlockSegmentPostings; - /// `SegmentPostings` represents the inverted list or postings associated to /// a term in a `Segment`. /// @@ -88,7 +83,7 @@ impl SegmentPostings { } let block_segment_postings = BlockSegmentPostings::open( docs.len() as u32, - FileSlice::new(buffer), + FileSlice::from(buffer), IndexRecordOption::Basic, IndexRecordOption::Basic, ) @@ -134,7 +129,7 @@ impl SegmentPostings { .unwrap(); let block_segment_postings = BlockSegmentPostings::open( doc_and_tfs.len() as u32, - FileSlice::new(buffer), + FileSlice::from(buffer), IndexRecordOption::WithFreqs, IndexRecordOption::WithFreqs, ) diff --git a/src/store/mod.rs b/src/store/mod.rs index abc3d9f13..7327d65fb 100644 --- a/src/store/mod.rs +++ b/src/store/mod.rs @@ -115,7 +115,7 @@ pub mod tests { #[test] fn test_store() -> crate::Result<()> { let path = Path::new("store"); - let mut directory = RAMDirectory::create(); + let directory = RAMDirectory::create(); let store_wrt = directory.open_write(path)?; let schema = write_lorem_ipsum_store(store_wrt, 1_000); let field_title = schema.get_field("title").unwrap(); @@ -149,7 +149,7 @@ mod bench { #[bench] #[cfg(feature = "mmap")] fn bench_store_encode(b: &mut Bencher) { - let mut directory = RAMDirectory::create(); + let directory = RAMDirectory::create(); let path = Path::new("store"); b.iter(|| { write_lorem_ipsum_store(directory.open_write(path).unwrap(), 1_000); @@ -159,7 +159,7 @@ mod bench { #[bench] fn bench_store_decode(b: &mut Bencher) { - let mut directory = RAMDirectory::create(); + let directory = RAMDirectory::create(); let path = Path::new("store"); write_lorem_ipsum_store(directory.open_write(path).unwrap(), 1_000); let store_file = directory.open_read(path).unwrap(); diff --git a/src/termdict/mod.rs b/src/termdict/mod.rs index 70136f4d2..4a2e4fe2b 100644 --- a/src/termdict/mod.rs +++ b/src/termdict/mod.rs @@ -69,7 +69,7 @@ mod tests { "Sweden", "Switzerland", ]; - let mut directory = RAMDirectory::create(); + let directory = RAMDirectory::create(); let path = PathBuf::from("TermDictionary"); { let write = directory.open_write(&path)?; @@ -92,7 +92,7 @@ mod tests { #[test] fn test_term_dictionary_simple() -> crate::Result<()> { - let mut directory = RAMDirectory::create(); + let directory = RAMDirectory::create(); let path = PathBuf::from("TermDictionary"); { let write = directory.open_write(&path)?; @@ -169,7 +169,7 @@ mod tests { } term_dictionary_builder.finish().unwrap() }; - let term_file = FileSlice::new(buffer); + let term_file = FileSlice::from(buffer); let term_dictionary: TermDictionary = TermDictionary::open(term_file)?; { let mut streamer = term_dictionary.stream(); @@ -200,7 +200,7 @@ mod tests { term_dictionary_builder.insert("abr", &make_term_info(2))?; term_dictionary_builder.finish()? }; - let term_dict_file = FileSlice::new(buffer); + let term_dict_file = FileSlice::from(buffer); let term_dictionary: TermDictionary = TermDictionary::open(term_dict_file)?; let mut kv_stream = term_dictionary.stream(); assert!(kv_stream.advance()); @@ -230,7 +230,7 @@ mod tests { term_dictionary_builder.finish().unwrap() }; - let file = FileSlice::new(buffer); + let file = FileSlice::from(buffer); let term_dictionary: TermDictionary = TermDictionary::open(file)?; { @@ -298,7 +298,7 @@ mod tests { .unwrap(); term_dictionary_builder.finish().unwrap() }; - let file = FileSlice::new(buffer); + let file = FileSlice::from(buffer); let term_dictionary: TermDictionary = TermDictionary::open(file)?; let mut stream = term_dictionary.stream(); assert!(stream.advance()); @@ -319,7 +319,7 @@ mod tests { } term_dictionary_builder.finish()? }; - let file = FileSlice::new(buffer); + let file = FileSlice::from(buffer); let term_dictionary: TermDictionary = TermDictionary::open(file)?; let value_list = |mut streamer: TermStreamer<'_>, backwards: bool| { @@ -426,7 +426,7 @@ mod tests { "Switzerland", ]; - let mut directory = RAMDirectory::create(); + let directory = RAMDirectory::create(); let path = PathBuf::from("TermDictionary"); { let write = directory.open_write(&path)?; diff --git a/src/termdict/term_info_store.rs b/src/termdict/term_info_store.rs index b61c41251..bf5049d1e 100644 --- a/src/termdict/term_info_store.rs +++ b/src/termdict/term_info_store.rs @@ -321,7 +321,7 @@ mod tests { } let mut buffer = Vec::new(); store_writer.serialize(&mut buffer)?; - let term_info_store = TermInfoStore::open(FileSlice::new(buffer))?; + let term_info_store = TermInfoStore::open(FileSlice::from(buffer))?; for i in 0..1000 { assert_eq!(term_info_store.get(i as u64), term_infos[i]); } diff --git a/src/termdict/termdict.rs b/src/termdict/termdict.rs index 9da4678d9..0dd54ec5d 100644 --- a/src/termdict/termdict.rs +++ b/src/termdict/termdict.rs @@ -98,7 +98,7 @@ static EMPTY_TERM_DICT_FILE: Lazy = Lazy::new(|| { .expect("Creating a TermDictionaryBuilder in a Vec should never fail") .finish() .expect("Writing in a Vec should never fail"); - FileSlice::new(term_dictionary_data) + FileSlice::from(term_dictionary_data) }); /// The term dictionary contains all of the terms in