rust/tantivy
1
0
Fork 0
mirror of https://github.com/quickwit-oss/tantivy.git synced 2026-06-02 16:40:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactoring dir (#905)

Browse Source
This commit is contained in:
Paul Masurel
2020-10-11 22:22:56 +09:00
committed by GitHub
parent 7a78b1cba3
commit 01b4aa9adc
19 changed files with 186 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/common/composite_file.rs
View File

@@ -196,7 +196,7 @@ mod test {
#[test]
fn test_composite_file() -> crate::Result<()> {
let path = Path::new("test_path");
let mut directory = RAMDirectory::create();
let directory = RAMDirectory::create();
{
let w = directory.open_write(path).unwrap();
let mut composite_write = CompositeWrite::wrap(w);

5
src/core/index.rs
View File

@@ -21,7 +21,6 @@ use crate::schema::FieldType;
use crate::schema::Schema;
use crate::tokenizer::{TextAnalyzer, TokenizerManager};
use crate::IndexWriter;
use std::borrow::BorrowMut;
use std::collections::HashSet;
use std::fmt;
@@ -153,8 +152,8 @@ impl Index {
/// Create a new index from a directory.
///
/// This will overwrite existing meta.json
fn from_directory(mut directory: ManagedDirectory, schema: Schema) -> crate::Result<Index> {
save_new_metas(schema.clone(), directory.borrow_mut())?;
fn from_directory(directory: ManagedDirectory, schema: Schema) -> crate::Result<Index> {
save_new_metas(schema.clone(), &directory)?;
let metas = IndexMeta::with_schema(schema);
Index::create_from_metas(directory, &metas, SegmentMetaInventory::default())
}

4
src/directory/directory.rs
View File

@@ -152,7 +152,7 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
/// was not called.
///
/// The file may not previously exist.
fn open_write(&mut self, path: &Path) -> Result<WritePtr, OpenWriteError>;
fn open_write(&self, path: &Path) -> Result<WritePtr, OpenWriteError>;
/// Reads the full content file that has been written using
/// atomic_write.
@@ -168,7 +168,7 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
/// a partially written file.
///
/// The file may or may not previously exist.
fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()>;
fn atomic_write(&self, path: &Path, data: &[u8]) -> io::Result<()>;
/// Acquire a lock in the given directory.
///

185
src/directory/file_slice.rs
View File

@@ -1,6 +1,7 @@
use stable_deref_trait::StableDeref;
use crate::common::HasLen;
use crate::directory::OwnedBytes;
use stable_deref_trait::{CloneStableDeref, StableDeref};
use std::sync::Arc;
use std::{io, ops::Deref};
@@ -8,57 +9,81 @@ pub type BoxedData = Box<dyn Deref<Target = [u8]> + Send + Sync + 'static>;
/// Objects that represents files sections in tantivy.
///
/// These read objects are only in charge to deliver
/// the data in the form of a constant read-only `&[u8]`.
/// Whatever happens to the directory file, the data
/// hold by this object should never be altered or destroyed.
pub trait FileSliceTrait: 'static + Send + Sync + HasLen {
fn read_bytes(&self) -> io::Result<OwnedBytes>;
fn slice(&self, from: usize, to: usize) -> FileSlice;
/// By contract, whatever happens to the directory file, as long as a FileHandle
/// is alive, the data associated with it cannot be altered or destroyed.
///
/// The underlying behavior is therefore specific to the `Directory` that created it.
/// Despite its name, a `FileSlice` may or may not directly map to an actual file
/// on the filesystem.
pub trait FileHandle: 'static + Send + Sync + HasLen {
fn read_bytes(&self, from: usize, to: usize) -> io::Result<OwnedBytes>;
}
impl FileSliceTrait for &'static [u8] {
fn read_bytes(&self) -> io::Result<OwnedBytes> {
Ok(OwnedBytes::new(*self))
}
fn slice(&self, from: usize, to: usize) -> FileSlice {
FileSlice::from(&self[from..to])
impl FileHandle for &'static [u8] {
fn read_bytes(&self, from: usize, to: usize) -> io::Result<OwnedBytes> {
let bytes = &self[from..to];
Ok(OwnedBytes::new(bytes))
}
}
impl HasLen for &'static [u8] {
impl<T: Deref<Target = [u8]>> HasLen for T {
fn len(&self) -> usize {
self.as_ref().len()
}
}
impl<B> From<B> for FileSlice
where
B: StableDeref + Deref<Target = [u8]> + 'static + Send + Sync,
{
fn from(bytes: B) -> FileSlice {
FileSlice::new(OwnedBytes::new(bytes))
}
}
/// Logical slice of read only file in tantivy.
//
/// In other words, it is more or less equivalent to the triplet `(file, start_byteoffset, stop_offset)`.
/// It can be cloned and sliced cheaply.
///
/// FileSlice is a simple wrapper over an `Arc<Box<dyn FileSliceTrait>>`. It can
/// be cloned cheaply.
///
/// The underlying behavior is therefore specific to the `Directory` that created it.
/// Despite its name, a `FileSlice` may or may not directly map to an actual file
/// on the filesystem.
#[derive(Clone)]
pub struct FileSlice(Arc<Box<dyn FileSliceTrait>>);
pub struct FileSlice {
data: Arc<Box<dyn FileHandle>>,
start: usize,
stop: usize,
}
impl FileSlice {
/// Creates a FileSlice, wrapping over a FileSliceTrait.
/// Wraps a new `Deref<Target = [u8]>`
pub fn new<D>(data: D) -> Self
where
D: Deref<Target = [u8]> + Send + Sync + 'static,
D: FileHandle,
{
FileSlice::from(SlicedDeref::new(data))
let len = data.len();
FileSlice {
data: Arc::new(Box::new(data)),
start: 0,
stop: len,
}
}
/// Creates a fileslice that is just a view over a slice of the data.
///
/// # Panics
/// Panics if `to < from` or if `to` exceeds the filesize.
pub fn slice(&self, from: usize, to: usize) -> FileSlice {
assert!(to <= self.len());
assert!(to >= from);
FileSlice {
data: self.data.clone(),
start: self.start + from,
stop: self.start + to,
}
}
/// Creates an empty FileSlice
pub fn empty() -> FileSlice {
let data: &'static [u8] = &[];
FileSlice::from(data)
const EMPTY_SLICE: &'static [u8] = &[];
FileSlice::from(EMPTY_SLICE)
}
/// Returns a `OwnedBytes` with all of the data in the `FileSlice`.
@@ -68,10 +93,10 @@ impl FileSlice {
/// In particular, it is up to the `Directory` implementation
/// to handle caching if needed.
pub fn read_bytes(&self) -> io::Result<OwnedBytes> {
self.0.read_bytes()
self.data.read_bytes(self.start, self.stop)
}
/// Splits the file slice at the given offset and return two file slices.
/// Splits the FileSlice at the given offset and return two file slices.
/// `file_slice[..split_offset]` and `file_slice[split_offset..]`.
///
/// This operation is cheap and must not copy any underlying data.
@@ -88,18 +113,6 @@ impl FileSlice {
self.split(left_len)
}
/// Creates a FileSlice that is just a view over a slice of the data.
pub fn slice(&self, start: usize, stop: usize) -> FileSlice {
assert!(
start <= stop,
"Requested negative slice [{}..{}]",
start,
stop
);
assert!(stop <= self.len());
self.0.slice(start, stop)
}
/// Like `.slice(...)` but enforcing only the `from`
/// boundary.
///
@@ -118,94 +131,14 @@ impl FileSlice {
}
impl HasLen for FileSlice {
fn len(&self) -> usize {
self.0.len()
}
}
impl<S: FileSliceTrait> From<S> for FileSlice {
fn from(file: S) -> Self {
FileSlice(Arc::new(Box::new(file)))
}
}
impl From<Arc<BoxedData>> for FileSlice {
fn from(data: Arc<BoxedData>) -> Self {
let slice_deref: SlicedDeref = SlicedDeref::from(data);
FileSlice::from(slice_deref)
}
}
/// `SliceDeref` wraps an `Arc<BoxData>` to implement `FileSliceTrait` .
/// It keeps track of (start, stop) boundaries.
#[derive(Clone)]
pub struct SlicedDeref {
data: Arc<BoxedData>,
start: usize,
stop: usize,
}
impl SlicedDeref {
/// Wraps a new `Deref<Target = [u8]>`
pub fn new<D>(data: D) -> Self
where
D: Deref<Target = [u8]> + 'static + Send + Sync,
{
let len = data.len();
SlicedDeref {
data: Arc::new(Box::new(data)),
start: 0,
stop: len,
}
}
}
impl From<Arc<BoxedData>> for SlicedDeref {
fn from(data: Arc<BoxedData>) -> Self {
let len = data.len();
SlicedDeref {
data,
start: 0,
stop: len,
}
}
}
unsafe impl StableDeref for SlicedDeref {}
unsafe impl CloneStableDeref for SlicedDeref {}
impl FileSliceTrait for SlicedDeref {
fn read_bytes(&self) -> io::Result<OwnedBytes> {
Ok(OwnedBytes::new(self.clone()))
}
fn slice(&self, from: usize, to: usize) -> FileSlice {
assert!(to <= self.len());
FileSlice::from(SlicedDeref {
data: self.data.clone(),
start: self.start + from,
stop: self.start + to,
})
}
}
impl HasLen for SlicedDeref {
fn len(&self) -> usize {
self.stop - self.start
}
}
impl Deref for SlicedDeref {
type Target = [u8];
fn deref(&self) -> &Self::Target {
&self.data.deref()[self.start..self.stop]
}
}
#[cfg(test)]
mod tests {
use super::{FileSlice, FileSliceTrait, SlicedDeref};
use super::{FileHandle, FileSlice};
use crate::common::HasLen;
use std::io;
@@ -249,13 +182,13 @@ mod tests {
#[test]
fn test_file_slice_trait_slice_len() {
let blop: &'static [u8] = b"abc";
let owned_bytes: Box<dyn FileSliceTrait> = Box::new(blop);
let owned_bytes: Box<dyn FileHandle> = Box::new(blop);
assert_eq!(owned_bytes.len(), 3);
}
#[test]
fn test_slice_deref() -> io::Result<()> {
let slice_deref = SlicedDeref::new(&b"abcdef"[..]);
let slice_deref = FileSlice::new(&b"abcdef"[..]);
assert_eq!(slice_deref.len(), 6);
assert_eq!(slice_deref.read_bytes()?.as_ref(), b"abcdef");
assert_eq!(slice_deref.slice(1, 4).read_bytes()?.as_ref(), b"bcd");

12
src/directory/managed_directory.rs
View File

@@ -53,7 +53,7 @@ struct MetaInformation {
/// Saves the file containing the list of existing files
/// that were created by tantivy.
fn save_managed_paths(
directory: &mut dyn Directory,
directory: &dyn Directory,
wlock: &RwLockWriteGuard<'_, MetaInformation>,
) -> io::Result<()> {
let mut w = serde_json::to_vec(&wlock.managed_paths)?;
@@ -212,7 +212,7 @@ impl ManagedDirectory {
/// File starting by "." are reserved to locks.
/// They are not managed and cannot be subjected
/// to garbage collection.
fn register_file_as_managed(&mut self, filepath: &Path) -> io::Result<()> {
fn register_file_as_managed(&self, filepath: &Path) -> io::Result<()> {
// Files starting by "." (e.g. lock files) are not managed.
if !is_managed(filepath) {
return Ok(());
@@ -223,7 +223,7 @@ impl ManagedDirectory {
.expect("Managed file lock poisoned");
let has_changed = meta_wlock.managed_paths.insert(filepath.to_owned());
if has_changed {
save_managed_paths(self.directory.as_mut(), &meta_wlock)?;
save_managed_paths(self.directory.as_ref(), &meta_wlock)?;
}
Ok(())
}
@@ -282,7 +282,7 @@ impl Directory for ManagedDirectory {
Ok(reader)
}
fn open_write(&mut self, path: &Path) -> result::Result<WritePtr, OpenWriteError> {
fn open_write(&self, path: &Path) -> result::Result<WritePtr, OpenWriteError> {
self.register_file_as_managed(path)
.map_err(|io_error| OpenWriteError::wrap_io_error(io_error, path.to_path_buf()))?;
Ok(io::BufWriter::new(Box::new(FooterProxy::new(
@@ -294,7 +294,7 @@ impl Directory for ManagedDirectory {
))))
}
fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()> {
fn atomic_write(&self, path: &Path, data: &[u8]) -> io::Result<()> {
self.register_file_as_managed(path)?;
self.directory.atomic_write(path, data)
}
@@ -416,7 +416,7 @@ mod tests_mmap_specific {
let tempdir_path = PathBuf::from(tempdir.path());
let mmap_directory = MmapDirectory::open(&tempdir_path)?;
let mut managed_directory = ManagedDirectory::wrap(mmap_directory)?;
let managed_directory = ManagedDirectory::wrap(mmap_directory)?;
let mut write = managed_directory.open_write(test_path1)?;
write.write_all(&[0u8, 1u8])?;
write.terminate()?;

32
src/directory/mmap_directory.rs
View File

@@ -17,7 +17,7 @@ use notify::RawEvent;
use notify::RecursiveMode;
use notify::Watcher;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use stable_deref_trait::StableDeref;
use std::convert::From;
use std::fmt;
use std::fs::OpenOptions;
@@ -32,6 +32,7 @@ use std::sync::Mutex;
use std::sync::RwLock;
use std::sync::Weak;
use std::thread;
use std::{collections::HashMap, ops::Deref};
use tempfile::TempDir;
/// Create a default io error given a string.
@@ -400,6 +401,18 @@ impl TerminatingWrite for SafeFileWriter {
}
}
#[derive(Clone)]
struct MmapArc(Arc<Box<dyn Deref<Target = [u8]> + Send + Sync>>);
impl Deref for MmapArc {
type Target = [u8];
fn deref(&self) -> &[u8] {
self.0.deref()
}
}
unsafe impl StableDeref for MmapArc {}
impl Directory for MmapDirectory {
fn open_read(&self, path: &Path) -> result::Result<FileSlice, OpenReadError> {
debug!("Open Read {:?}", path);
@@ -414,10 +427,11 @@ impl Directory for MmapDirectory {
let io_err = make_io_err(msg);
OpenReadError::wrap_io_error(io_err, path.to_path_buf())
})?;
Ok(mmap_cache
.get_mmap(&full_path)?
.map(FileSlice::from)
.unwrap_or_else(FileSlice::empty))
if let Some(mmap_arc) = mmap_cache.get_mmap(&full_path)? {
Ok(FileSlice::from(MmapArc(mmap_arc)))
} else {
Ok(FileSlice::empty())
}
}
/// Any entry associated to the path in the mmap will be
@@ -447,7 +461,7 @@ impl Directory for MmapDirectory {
full_path.exists()
}
fn open_write(&mut self, path: &Path) -> Result<WritePtr, OpenWriteError> {
fn open_write(&self, path: &Path) -> Result<WritePtr, OpenWriteError> {
debug!("Open Write {:?}", path);
let full_path = self.resolve_path(path);
@@ -497,7 +511,7 @@ impl Directory for MmapDirectory {
}
}
fn atomic_write(&mut self, path: &Path, content: &[u8]) -> io::Result<()> {
fn atomic_write(&self, path: &Path, content: &[u8]) -> io::Result<()> {
debug!("Atomic Write {:?}", path);
let mut tempfile = tempfile::Builder::new().tempfile_in(&self.inner.root_path)?;
tempfile.write_all(content)?;
@@ -557,7 +571,7 @@ mod tests {
// cannot be mmapped.
//
// In that case the directory returns a SharedVecSlice.
let mut mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
let mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
let path = PathBuf::from("test");
{
let mut w = mmap_directory.open_write(&path).unwrap();
@@ -573,7 +587,7 @@ mod tests {
// here we test if the cache releases
// mmaps correctly.
let mut mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
let mmap_directory = MmapDirectory::create_from_tempdir().unwrap();
let num_paths = 10;
let paths: Vec<PathBuf> = (0..num_paths)
.map(|i| PathBuf::from(&*format!("file_{}", i)))

19
src/directory/owned_bytes.rs
View File

@@ -4,6 +4,8 @@ use std::ops::Deref;
use std::sync::Arc;
use std::{fmt, io};
use super::file_slice::FileHandle;
/// An OwnedBytes simply wraps an object that owns a slice of data and exposes
/// this data as a static slice.
///
@@ -14,6 +16,12 @@ pub struct OwnedBytes {
box_stable_deref: Arc<dyn Deref<Target = [u8]> + Sync + Send>,
}
impl FileHandle for OwnedBytes {
fn read_bytes(&self, from: usize, to: usize) -> io::Result<OwnedBytes> {
Ok(self.slice(from, to))
}
}
impl OwnedBytes {
/// Creates an empty `OwnedBytes`.
pub fn empty() -> OwnedBytes {
@@ -25,13 +33,22 @@ impl OwnedBytes {
data_holder: T,
) -> OwnedBytes {
let box_stable_deref = Arc::new(data_holder);
let data = unsafe { mem::transmute::<_, &'static [u8]>(box_stable_deref.deref().deref()) };
let bytes: &[u8] = box_stable_deref.as_ref();
let data = unsafe { mem::transmute::<_, &'static [u8]>(bytes.deref()) };
OwnedBytes {
box_stable_deref,
data,
}
}
/// creates a fileslice that is just a view over a slice of the data.
pub fn slice(&self, from: usize, to: usize) -> Self {
OwnedBytes {
data: &self.data[from..to],
box_stable_deref: self.box_stable_deref.clone(),
}
}
/// Returns the underlying slice of data.
/// `Deref` and `AsRef` are also available.
#[inline(always)]

14
src/directory/ram_directory.rs
View File

@@ -86,7 +86,7 @@ struct InnerDirectory {
impl InnerDirectory {
fn write(&mut self, path: PathBuf, data: &[u8]) -> bool {
let data = FileSlice::new(Vec::from(data));
let data = FileSlice::from(data.to_vec());
self.fs.insert(path, data).is_some()
}
@@ -151,7 +151,7 @@ impl RAMDirectory {
/// written using the `atomic_write` api.
///
/// If an error is encounterred, files may be persisted partially.
pub fn persist(&self, dest: &mut dyn Directory) -> crate::Result<()> {
pub fn persist(&self, dest: &dyn Directory) -> crate::Result<()> {
let wlock = self.fs.write().unwrap();
for (path, file) in wlock.fs.iter() {
let mut dest_wrt = dest.open_write(path)?;
@@ -181,7 +181,7 @@ impl Directory for RAMDirectory {
self.fs.read().unwrap().exists(path)
}
fn open_write(&mut self, path: &Path) -> Result<WritePtr, OpenWriteError> {
fn open_write(&self, path: &Path) -> Result<WritePtr, OpenWriteError> {
let mut fs = self.fs.write().unwrap();
let path_buf = PathBuf::from(path);
let vec_writer = VecWriter::new(path_buf.clone(), self.clone());
@@ -205,7 +205,7 @@ impl Directory for RAMDirectory {
Ok(bytes.as_slice().to_owned())
}
fn atomic_write(&mut self, path: &Path, data: &[u8]) -> io::Result<()> {
fn atomic_write(&self, path: &Path, data: &[u8]) -> io::Result<()> {
fail_point!("RAMDirectory::atomic_write", |msg| Err(io::Error::new(
io::ErrorKind::Other,
msg.unwrap_or_else(|| "Undefined".to_string())
@@ -242,13 +242,13 @@ mod tests {
let msg_seq: &'static [u8] = b"sequential is the way";
let path_atomic: &'static Path = Path::new("atomic");
let path_seq: &'static Path = Path::new("seq");
let mut directory = RAMDirectory::create();
let directory = RAMDirectory::create();
assert!(directory.atomic_write(path_atomic, msg_atomic).is_ok());
let mut wrt = directory.open_write(path_seq).unwrap();
assert!(wrt.write_all(msg_seq).is_ok());
assert!(wrt.flush().is_ok());
let mut directory_copy = RAMDirectory::create();
assert!(directory.persist(&mut directory_copy).is_ok());
let directory_copy = RAMDirectory::create();
assert!(directory.persist(&directory_copy).is_ok());
assert_eq!(directory_copy.atomic_read(path_atomic).unwrap(), msg_atomic);
assert_eq!(directory_copy.atomic_read(path_seq).unwrap(), msg_seq);
}

72
src/directory/tests.rs
View File

@@ -21,46 +21,46 @@ mod mmap_directory_tests {
#[test]
fn test_simple() -> crate::Result<()> {
let mut directory = make_directory();
super::test_simple(&mut directory)
let directory = make_directory();
super::test_simple(&directory)
}
#[test]
fn test_write_create_the_file() {
let mut directory = make_directory();
super::test_write_create_the_file(&mut directory);
let directory = make_directory();
super::test_write_create_the_file(&directory);
}
#[test]
fn test_rewrite_forbidden() -> crate::Result<()> {
let mut directory = make_directory();
super::test_rewrite_forbidden(&mut directory)?;
let directory = make_directory();
super::test_rewrite_forbidden(&directory)?;
Ok(())
}
#[test]
fn test_directory_delete() -> crate::Result<()> {
let mut directory = make_directory();
super::test_directory_delete(&mut directory)?;
let directory = make_directory();
super::test_directory_delete(&directory)?;
Ok(())
}
#[test]
fn test_lock_non_blocking() {
let mut directory = make_directory();
super::test_lock_non_blocking(&mut directory);
let directory = make_directory();
super::test_lock_non_blocking(&directory);
}
#[test]
fn test_lock_blocking() {
let mut directory = make_directory();
super::test_lock_blocking(&mut directory);
let directory = make_directory();
super::test_lock_blocking(&directory);
}
#[test]
fn test_watch() {
let mut directory = make_directory();
super::test_watch(&mut directory);
let directory = make_directory();
super::test_watch(&directory);
}
}
@@ -75,46 +75,46 @@ mod ram_directory_tests {
#[test]
fn test_simple() -> crate::Result<()> {
let mut directory = make_directory();
super::test_simple(&mut directory)
let directory = make_directory();
super::test_simple(&directory)
}
#[test]
fn test_write_create_the_file() {
let mut directory = make_directory();
super::test_write_create_the_file(&mut directory);
let directory = make_directory();
super::test_write_create_the_file(&directory);
}
#[test]
fn test_rewrite_forbidden() -> crate::Result<()> {
let mut directory = make_directory();
super::test_rewrite_forbidden(&mut directory)?;
let directory = make_directory();
super::test_rewrite_forbidden(&directory)?;
Ok(())
}
#[test]
fn test_directory_delete() -> crate::Result<()> {
let mut directory = make_directory();
super::test_directory_delete(&mut directory)?;
let directory = make_directory();
super::test_directory_delete(&directory)?;
Ok(())
}
#[test]
fn test_lock_non_blocking() {
let mut directory = make_directory();
super::test_lock_non_blocking(&mut directory);
let directory = make_directory();
super::test_lock_non_blocking(&directory);
}
#[test]
fn test_lock_blocking() {
let mut directory = make_directory();
super::test_lock_blocking(&mut directory);
let directory = make_directory();
super::test_lock_blocking(&directory);
}
#[test]
fn test_watch() {
let mut directory = make_directory();
super::test_watch(&mut directory);
let directory = make_directory();
super::test_watch(&directory);
}
}
@@ -122,12 +122,12 @@ mod ram_directory_tests {
#[should_panic]
fn ram_directory_panics_if_flush_forgotten() {
let test_path: &'static Path = Path::new("some_path_for_test");
let mut ram_directory = RAMDirectory::create();
let ram_directory = RAMDirectory::create();
let mut write_file = ram_directory.open_write(test_path).unwrap();
assert!(write_file.write_all(&[4]).is_ok());
}
fn test_simple(directory: &mut dyn Directory) -> crate::Result<()> {
fn test_simple(directory: &dyn Directory) -> crate::Result<()> {
let test_path: &'static Path = Path::new("some_path_for_test");
let mut write_file = directory.open_write(test_path)?;
assert!(directory.exists(test_path));
@@ -143,7 +143,7 @@ fn test_simple(directory: &mut dyn Directory) -> crate::Result<()> {
Ok(())
}
fn test_rewrite_forbidden(directory: &mut dyn Directory) -> crate::Result<()> {
fn test_rewrite_forbidden(directory: &dyn Directory) -> crate::Result<()> {
let test_path: &'static Path = Path::new("some_path_for_test");
directory.open_write(test_path)?;
assert!(directory.exists(test_path));
@@ -152,7 +152,7 @@ fn test_rewrite_forbidden(directory: &mut dyn Directory) -> crate::Result<()> {
Ok(())
}
fn test_write_create_the_file(directory: &mut dyn Directory) {
fn test_write_create_the_file(directory: &dyn Directory) {
let test_path: &'static Path = Path::new("some_path_for_test");
{
assert!(directory.open_read(test_path).is_err());
@@ -163,7 +163,7 @@ fn test_write_create_the_file(directory: &mut dyn Directory) {
}
}
fn test_directory_delete(directory: &mut dyn Directory) -> crate::Result<()> {
fn test_directory_delete(directory: &dyn Directory) -> crate::Result<()> {
let test_path: &'static Path = Path::new("some_path_for_test");
assert!(directory.open_read(test_path).is_err());
let mut write_file = directory.open_write(&test_path)?;
@@ -189,7 +189,7 @@ fn test_directory_delete(directory: &mut dyn Directory) -> crate::Result<()> {
Ok(())
}
fn test_watch(directory: &mut dyn Directory) {
fn test_watch(directory: &dyn Directory) {
let num_progress: Arc<AtomicUsize> = Default::default();
let counter: Arc<AtomicUsize> = Default::default();
let counter_clone = counter.clone();
@@ -224,7 +224,7 @@ fn test_watch(directory: &mut dyn Directory) {
assert!(10 <= counter.load(SeqCst));
}
fn test_lock_non_blocking(directory: &mut dyn Directory) {
fn test_lock_non_blocking(directory: &dyn Directory) {
{
let lock_a_res = directory.acquire_lock(&Lock {
filepath: PathBuf::from("a.lock"),
@@ -249,7 +249,7 @@ fn test_lock_non_blocking(directory: &mut dyn Directory) {
assert!(lock_a_res.is_ok());
}
fn test_lock_blocking(directory: &mut dyn Directory) {
fn test_lock_blocking(directory: &dyn Directory) {
let lock_a_res = directory.acquire_lock(&Lock {
filepath: PathBuf::from("a.lock"),
is_blocking: true,

2
src/fastfield/delete.rs
View File

@@ -54,7 +54,7 @@ impl DeleteBitSet {
for &doc in docs {
bitset.insert(doc);
}
let mut directory = RAMDirectory::create();
let directory = RAMDirectory::create();
let path = Path::new("dummydeletebitset");
let mut wrt = directory.open_write(path).unwrap();
write_delete_bitset(&bitset, max_doc, &mut wrt).unwrap();

18
src/fastfield/mod.rs
View File

@@ -242,7 +242,7 @@ mod tests {
#[test]
fn test_intfastfield_small() -> crate::Result<()> {
let path = Path::new("test");
let mut directory: RAMDirectory = RAMDirectory::create();
let directory: RAMDirectory = RAMDirectory::create();
{
let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
let mut serializer = FastFieldSerializer::from_write(write).unwrap();
@@ -269,7 +269,7 @@ mod tests {
#[test]
fn test_intfastfield_large() -> crate::Result<()> {
let path = Path::new("test");
let mut directory: RAMDirectory = RAMDirectory::create();
let directory: RAMDirectory = RAMDirectory::create();
{
let write: WritePtr = directory.open_write(Path::new("test"))?;
let mut serializer = FastFieldSerializer::from_write(write)?;
@@ -308,7 +308,7 @@ mod tests {
#[test]
fn test_intfastfield_null_amplitude() -> crate::Result<()> {
let path = Path::new("test");
let mut directory: RAMDirectory = RAMDirectory::create();
let directory: RAMDirectory = RAMDirectory::create();
{
let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
@@ -338,7 +338,7 @@ mod tests {
#[test]
fn test_intfastfield_large_numbers() -> crate::Result<()> {
let path = Path::new("test");
let mut directory: RAMDirectory = RAMDirectory::create();
let directory: RAMDirectory = RAMDirectory::create();
{
let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
@@ -374,7 +374,7 @@ mod tests {
#[test]
fn test_signed_intfastfield() -> crate::Result<()> {
let path = Path::new("test");
let mut directory: RAMDirectory = RAMDirectory::create();
let directory: RAMDirectory = RAMDirectory::create();
let mut schema_builder = Schema::builder();
let i64_field = schema_builder.add_i64_field("field", FAST);
@@ -417,7 +417,7 @@ mod tests {
#[test]
fn test_signed_intfastfield_default_val() -> crate::Result<()> {
let path = Path::new("test");
let mut directory: RAMDirectory = RAMDirectory::create();
let directory: RAMDirectory = RAMDirectory::create();
let mut schema_builder = Schema::builder();
let i64_field = schema_builder.add_i64_field("field", FAST);
let schema = schema_builder.build();
@@ -456,7 +456,7 @@ mod tests {
let path = Path::new("test");
let permutation = generate_permutation();
let n = permutation.len();
let mut directory = RAMDirectory::create();
let directory = RAMDirectory::create();
{
let write: WritePtr = directory.open_write(Path::new("test"))?;
let mut serializer = FastFieldSerializer::from_write(write)?;
@@ -612,7 +612,7 @@ mod bench {
fn bench_intfastfield_linear_fflookup(b: &mut Bencher) {
let path = Path::new("test");
let permutation = generate_permutation();
let mut directory: RAMDirectory = RAMDirectory::create();
let directory: RAMDirectory = RAMDirectory::create();
{
let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
let mut serializer = FastFieldSerializer::from_write(write).unwrap();
@@ -646,7 +646,7 @@ mod bench {
fn bench_intfastfield_fflookup(b: &mut Bencher) {
let path = Path::new("test");
let permutation = generate_permutation();
let mut directory: RAMDirectory = RAMDirectory::create();
let directory: RAMDirectory = RAMDirectory::create();
{
let write: WritePtr = directory.open_write(Path::new("test")).unwrap();
let mut serializer = FastFieldSerializer::from_write(write).unwrap();

2
src/fastfield/reader.rs
View File

@@ -127,7 +127,7 @@ impl<Item: FastValue> From<Vec<Item>> for FastFieldReader<Item> {
let field = schema_builder.add_u64_field("field", FAST);
let schema = schema_builder.build();
let path = Path::new("__dummy__");
let mut directory: RAMDirectory = RAMDirectory::create();
let directory: RAMDirectory = RAMDirectory::create();
{
let write: WritePtr = directory
.open_write(path)

4
src/indexer/segment_updater.rs
View File

@@ -43,7 +43,7 @@ const NUM_MERGE_THREADS: usize = 4;
/// and flushed.
///
/// This method is not part of tantivy's public API
pub fn save_new_metas(schema: Schema, directory: &mut dyn Directory) -> crate::Result<()> {
pub fn save_new_metas(schema: Schema, directory: &dyn Directory) -> crate::Result<()> {
save_metas(
&IndexMeta {
segments: Vec::new(),
@@ -64,7 +64,7 @@ pub fn save_new_metas(schema: Schema, directory: &mut dyn Directory) -> crate::R
/// and flushed.
///
/// This method is not part of tantivy's public API
fn save_metas(metas: &IndexMeta, directory: &mut dyn Directory) -> crate::Result<()> {
fn save_metas(metas: &IndexMeta, directory: &dyn Directory) -> crate::Result<()> {
info!("save metas");
let mut buffer = serde_json::to_vec_pretty(metas)?;
// Just adding a new line at the end of the buffer.

2
src/positions/mod.rs
View File

@@ -53,7 +53,7 @@ pub mod tests {
}
serializer.close().unwrap();
}
(FileSlice::new(stream_buffer), FileSlice::new(skip_buffer))
(FileSlice::from(stream_buffer), FileSlice::from(skip_buffer))
}
#[test]

15
src/postings/segment_postings.rs
View File

@@ -1,21 +1,16 @@
use crate::common::HasLen;
use crate::directory::FileSlice;
use crate::docset::DocSet;
use crate::fastfield::DeleteBitSet;
use crate::positions::PositionReader;
use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
use crate::postings::serializer::PostingsSerializer;
use crate::postings::BlockSearcher;
use crate::postings::BlockSegmentPostings;
use crate::postings::Postings;
use crate::schema::IndexRecordOption;
use crate::{DocId, TERMINATED};
use crate::directory::FileSlice;
use crate::fastfield::DeleteBitSet;
use crate::postings::BlockSegmentPostings;
/// `SegmentPostings` represents the inverted list or postings associated to
/// a term in a `Segment`.
///
@@ -88,7 +83,7 @@ impl SegmentPostings {
}
let block_segment_postings = BlockSegmentPostings::open(
docs.len() as u32,
FileSlice::new(buffer),
FileSlice::from(buffer),
IndexRecordOption::Basic,
IndexRecordOption::Basic,
)
@@ -134,7 +129,7 @@ impl SegmentPostings {
.unwrap();
let block_segment_postings = BlockSegmentPostings::open(
doc_and_tfs.len() as u32,
FileSlice::new(buffer),
FileSlice::from(buffer),
IndexRecordOption::WithFreqs,
IndexRecordOption::WithFreqs,
)

6
src/store/mod.rs
View File

@@ -115,7 +115,7 @@ pub mod tests {
#[test]
fn test_store() -> crate::Result<()> {
let path = Path::new("store");
let mut directory = RAMDirectory::create();
let directory = RAMDirectory::create();
let store_wrt = directory.open_write(path)?;
let schema = write_lorem_ipsum_store(store_wrt, 1_000);
let field_title = schema.get_field("title").unwrap();
@@ -149,7 +149,7 @@ mod bench {
#[bench]
#[cfg(feature = "mmap")]
fn bench_store_encode(b: &mut Bencher) {
let mut directory = RAMDirectory::create();
let directory = RAMDirectory::create();
let path = Path::new("store");
b.iter(|| {
write_lorem_ipsum_store(directory.open_write(path).unwrap(), 1_000);
@@ -159,7 +159,7 @@ mod bench {
#[bench]
fn bench_store_decode(b: &mut Bencher) {
let mut directory = RAMDirectory::create();
let directory = RAMDirectory::create();
let path = Path::new("store");
write_lorem_ipsum_store(directory.open_write(path).unwrap(), 1_000);
let store_file = directory.open_read(path).unwrap();

16
src/termdict/mod.rs
View File

@@ -69,7 +69,7 @@ mod tests {
"Sweden",
"Switzerland",
];
let mut directory = RAMDirectory::create();
let directory = RAMDirectory::create();
let path = PathBuf::from("TermDictionary");
{
let write = directory.open_write(&path)?;
@@ -92,7 +92,7 @@ mod tests {
#[test]
fn test_term_dictionary_simple() -> crate::Result<()> {
let mut directory = RAMDirectory::create();
let directory = RAMDirectory::create();
let path = PathBuf::from("TermDictionary");
{
let write = directory.open_write(&path)?;
@@ -169,7 +169,7 @@ mod tests {
}
term_dictionary_builder.finish().unwrap()
};
let term_file = FileSlice::new(buffer);
let term_file = FileSlice::from(buffer);
let term_dictionary: TermDictionary = TermDictionary::open(term_file)?;
{
let mut streamer = term_dictionary.stream();
@@ -200,7 +200,7 @@ mod tests {
term_dictionary_builder.insert("abr", &make_term_info(2))?;
term_dictionary_builder.finish()?
};
let term_dict_file = FileSlice::new(buffer);
let term_dict_file = FileSlice::from(buffer);
let term_dictionary: TermDictionary = TermDictionary::open(term_dict_file)?;
let mut kv_stream = term_dictionary.stream();
assert!(kv_stream.advance());
@@ -230,7 +230,7 @@ mod tests {
term_dictionary_builder.finish().unwrap()
};
let file = FileSlice::new(buffer);
let file = FileSlice::from(buffer);
let term_dictionary: TermDictionary = TermDictionary::open(file)?;
{
@@ -298,7 +298,7 @@ mod tests {
.unwrap();
term_dictionary_builder.finish().unwrap()
};
let file = FileSlice::new(buffer);
let file = FileSlice::from(buffer);
let term_dictionary: TermDictionary = TermDictionary::open(file)?;
let mut stream = term_dictionary.stream();
assert!(stream.advance());
@@ -319,7 +319,7 @@ mod tests {
}
term_dictionary_builder.finish()?
};
let file = FileSlice::new(buffer);
let file = FileSlice::from(buffer);
let term_dictionary: TermDictionary = TermDictionary::open(file)?;
let value_list = |mut streamer: TermStreamer<'_>, backwards: bool| {
@@ -426,7 +426,7 @@ mod tests {
"Switzerland",
];
let mut directory = RAMDirectory::create();
let directory = RAMDirectory::create();
let path = PathBuf::from("TermDictionary");
{
let write = directory.open_write(&path)?;

2
src/termdict/term_info_store.rs
View File

@@ -321,7 +321,7 @@ mod tests {
}
let mut buffer = Vec::new();
store_writer.serialize(&mut buffer)?;
let term_info_store = TermInfoStore::open(FileSlice::new(buffer))?;
let term_info_store = TermInfoStore::open(FileSlice::from(buffer))?;
for i in 0..1000 {
assert_eq!(term_info_store.get(i as u64), term_infos[i]);
}

2
src/termdict/termdict.rs
View File

@@ -98,7 +98,7 @@ static EMPTY_TERM_DICT_FILE: Lazy<FileSlice> = Lazy::new(|| {
.expect("Creating a TermDictionaryBuilder in a Vec<u8> should never fail")
.finish()
.expect("Writing in a Vec<u8> should never fail");
FileSlice::new(term_dictionary_data)
FileSlice::from(term_dictionary_data)
});
/// The term dictionary contains all of the terms in